├── .ci ├── assets │ ├── .gitkeep │ ├── bindings │ │ └── .gitkeep │ ├── release_requirements.txt │ └── httpie │ │ └── config.json ├── ansible │ ├── ansible.cfg │ ├── inventory.yaml │ ├── smash-config.json │ ├── filter │ │ └── repr.py │ ├── Containerfile.j2 │ ├── settings.py.j2 │ ├── build_container.yaml │ └── start_container.yaml └── scripts │ ├── check_gettext.sh │ ├── check_pulpcore_imports.sh │ ├── schema.py │ ├── update_redmine.sh │ ├── redmine.py │ ├── cherrypick.sh │ └── validate_commit_message.py ├── pulp_python ├── app │ ├── pypi │ │ ├── __init__.py │ │ └── serializers.py │ ├── migrations │ │ ├── __init__.py │ │ ├── 0006_pythonrepository_autopublish.py │ │ ├── 0009_pythondistribution_allow_uploads.py │ │ ├── 0002_pythonpackagecontent_python_version.py │ │ ├── 0007_pythonpackagecontent_mv-2-1.py │ │ ├── 0008_pythonpackagecontent_unique_sha256.py │ │ ├── 0003_new_sync_filters.py │ │ ├── 0005_pythonpackagecontent_sha256.py │ │ ├── 0004_DATA_swap_distribution_model.py │ │ └── 0001_initial.py │ ├── webserver_snippets │ │ ├── __init__.py │ │ ├── apache.conf │ │ └── nginx.conf │ ├── settings.py │ ├── tasks │ │ ├── __init__.py │ │ ├── publish.py │ │ └── upload.py │ ├── __init__.py │ ├── urls.py │ ├── models.py │ └── viewsets.py ├── tests │ ├── unit │ │ ├── __init__.py │ │ └── test_models.py │ ├── functional │ │ ├── __init__.py │ │ └── api │ │ │ ├── __init__.py │ │ │ ├── test_full_mirror.py │ │ │ ├── test_auto_publish.py │ │ │ ├── test_download_content.py │ │ │ └── test_consume_content.py │ └── __init__.py └── __init__.py ├── CHANGES ├── .gitignore ├── 402.removal ├── 400.bugfix └── .TEMPLATE.rst ├── unittest_requirements.txt ├── docs ├── contributing.rst ├── _scripts │ ├── index.sh │ ├── repo.sh │ ├── remote.sh │ ├── twine.sh │ ├── pip.sh │ ├── distribution.sh │ ├── artifact.sh │ ├── add_content_repo.sh │ ├── autoupdate.sh │ ├── upload.sh │ ├── sync.sh │ ├── publication.sh │ ├── clean.sh │ ├── base.sh │ └── quickstart.sh ├── changes.rst ├── restapi │ └── index.rst ├── _static │ └── survey_banner.js ├── tech-preview.rst ├── _templates │ └── restapi.html ├── workflows │ ├── index.rst │ ├── pypi.rst │ ├── publish.rst │ ├── upload.rst │ └── sync.rst ├── index.rst ├── installation.rst ├── Makefile └── conf.py ├── requirements.txt ├── functest_requirements.txt ├── shelf_reader-0.1-py2-none-any.whl ├── .github └── workflows │ ├── scripts │ ├── post_docs_test.sh │ ├── create_release_from_tag.sh │ ├── secrets.py │ ├── utils.sh │ ├── push_branch_and_tag_to_github.sh │ ├── check_commit.sh │ ├── publish_plugin_pypi.sh │ ├── publish_client_gem.sh │ ├── publish_docs.sh │ ├── publish_client_pypi.sh │ ├── before_script.sh │ ├── install_python_client.sh │ ├── install_ruby_client.sh │ ├── install.sh │ ├── script.sh │ ├── before_install.sh │ ├── release.py │ └── docs-publisher.py │ ├── ci.yml │ ├── nightly.yml │ └── release.yml ├── doc_requirements.txt ├── test_requirements.txt ├── dev_requirements.txt ├── .pep8speaks.yml ├── MANIFEST.in ├── .bumpversion.cfg ├── COPYRIGHT ├── pyproject.toml ├── flake8.cfg ├── .gitignore ├── HISTORY.rst ├── setup.py ├── template_config.yml ├── README.md ├── COMMITMENT ├── CONTRIBUTING.rst └── CHANGES.rst /.ci/assets/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.ci/assets/bindings/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pulp_python/app/pypi/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /CHANGES/.gitignore: -------------------------------------------------------------------------------- 1 | !.gitignore 2 | -------------------------------------------------------------------------------- /pulp_python/app/migrations/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pulp_python/tests/unit/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /unittest_requirements.txt: -------------------------------------------------------------------------------- 1 | mock 2 | -------------------------------------------------------------------------------- /pulp_python/tests/functional/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pulp_python/app/webserver_snippets/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pulp_python/tests/functional/api/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /CHANGES/402.removal: -------------------------------------------------------------------------------- 1 | Dropped support for Python < 3.8. 2 | -------------------------------------------------------------------------------- /docs/contributing.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../CONTRIBUTING.rst 2 | -------------------------------------------------------------------------------- /pulp_python/app/settings.py: -------------------------------------------------------------------------------- 1 | PYTHON_GROUP_UPLOADS = False 2 | -------------------------------------------------------------------------------- /pulp_python/tests/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Tests for Pulp Python. 3 | """ 4 | -------------------------------------------------------------------------------- /CHANGES/400.bugfix: -------------------------------------------------------------------------------- 1 | Fixed twine upload failing when using remote storage backends 2 | -------------------------------------------------------------------------------- /pulp_python/__init__.py: -------------------------------------------------------------------------------- 1 | default_app_config = 'pulp_python.app.PulpPythonPluginAppConfig' 2 | -------------------------------------------------------------------------------- /.ci/assets/release_requirements.txt: -------------------------------------------------------------------------------- 1 | bump2version 2 | gitpython 3 | python-redmine 4 | towncrier 5 | -------------------------------------------------------------------------------- /docs/_scripts/index.sh: -------------------------------------------------------------------------------- 1 | pulp python distribution create --name my-pypi --base-path my-pypi --repository foo 2 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | pulpcore>=3.13 2 | pkginfo 3 | packaging 4 | bandersnatch==4.4.0 5 | google-cloud-pubsub 6 | -------------------------------------------------------------------------------- /functest_requirements.txt: -------------------------------------------------------------------------------- 1 | git+https://github.com/pulp/pulp-smash.git#egg=pulp-smash 2 | pytest 3 | lxml 4 | twine 5 | -------------------------------------------------------------------------------- /docs/_scripts/repo.sh: -------------------------------------------------------------------------------- 1 | # Start by creating a new repository named "foo": 2 | pulp python repository create --name foo 3 | -------------------------------------------------------------------------------- /docs/changes.rst: -------------------------------------------------------------------------------- 1 | .. _pulp_python-changes: 2 | 3 | .. include:: ../CHANGES.rst 4 | 5 | .. include:: ../HISTORY.rst 6 | -------------------------------------------------------------------------------- /shelf_reader-0.1-py2-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/replit/pulp_python/master/shelf_reader-0.1-py2-none-any.whl -------------------------------------------------------------------------------- /pulp_python/app/webserver_snippets/apache.conf: -------------------------------------------------------------------------------- 1 | ProxyPass /pulp_python/pypi ${pulp-api}/pypi 2 | ProxyPassReverse /pulp_python/pypi ${pulp-api}/pypi 3 | -------------------------------------------------------------------------------- /.ci/assets/httpie/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "default_options": [ 3 | "--ignore-stdin", 4 | "--pretty=format", 5 | "--traceback" 6 | ] 7 | } 8 | -------------------------------------------------------------------------------- /.ci/ansible/ansible.cfg: -------------------------------------------------------------------------------- 1 | [defaults] 2 | inventory = inventory.yaml 3 | filter_plugins = filter 4 | retry_files_enabled = False 5 | transport = local 6 | nocows = 1 7 | stdout_callback = yaml 8 | -------------------------------------------------------------------------------- /.github/workflows/scripts/post_docs_test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | export BASE_ADDR=http://pulp:80 4 | export CONTENT_ADDR=http://pulp:80 5 | 6 | cd docs/_scripts/ 7 | bash ./quickstart.sh 8 | -------------------------------------------------------------------------------- /docs/_scripts/remote.sh: -------------------------------------------------------------------------------- 1 | # Create a remote that syncs some versions of shelf-reader into your repository. 2 | pulp python remote create --name bar --url https://pypi.org/ --includes '["shelf-reader"]' 3 | -------------------------------------------------------------------------------- /doc_requirements.txt: -------------------------------------------------------------------------------- 1 | build 2 | coreapi 3 | django 4 | djangorestframework 5 | django-filter 6 | drf-nested-routers 7 | plantuml 8 | pyyaml 9 | sphinx 10 | sphinx-rtd-theme 11 | sphinxcontrib-openapi 12 | towncrier 13 | -------------------------------------------------------------------------------- /test_requirements.txt: -------------------------------------------------------------------------------- 1 | coveralls 2 | coverage 3 | flake8 4 | flake8-docstrings 5 | flake8-tuple 6 | flake8-quotes 7 | mock 8 | git+https://github.com/pulp/pulp-smash.git#egg=pulp-smash 9 | pydocstyle<4 10 | pytest 11 | -------------------------------------------------------------------------------- /pulp_python/app/tasks/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Asynchronous task definitions. 3 | """ 4 | 5 | from .publish import publish # noqa:F401 6 | from .sync import sync # noqa:F401 7 | from .upload import upload, upload_group # noqa:F401 8 | -------------------------------------------------------------------------------- /.ci/ansible/inventory.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | all: 3 | children: 4 | containers: 5 | hosts: 6 | pulp: 7 | pulp-fixtures: 8 | minio: 9 | vars: 10 | ansible_connection: docker 11 | ... 12 | -------------------------------------------------------------------------------- /docs/_scripts/twine.sh: -------------------------------------------------------------------------------- 1 | # Build custom package 2 | python -m build $PLUGIN_SOURCE 3 | # Upload built package distributions to my-pypi 4 | twine upload --repository-url $BASE_ADDR/pypi/my-pypi/simple/ -u admin -p password "$PLUGIN_SOURCE"dist/* 5 | -------------------------------------------------------------------------------- /dev_requirements.txt: -------------------------------------------------------------------------------- 1 | black 2 | check-manifest 3 | flake8 4 | flake8-docstrings 5 | flake8-tuple 6 | flake8-quotes 7 | # pin pydocstyle until https://gitlab.com/pycqa/flake8-docstrings/issues/36 is resolved 8 | pydocstyle<4 9 | requests 10 | -------------------------------------------------------------------------------- /docs/_scripts/pip.sh: -------------------------------------------------------------------------------- 1 | echo 'pip install --trusted-host pulp -i $BASE_ADDR/pypi/foo/simple/ shelf-reader' 2 | pip install --trusted-host pulp -i $BASE_ADDR/pypi/foo/simple/ shelf-reader 3 | 4 | echo "is shelf reader installed?" 5 | pip list | grep shelf-reader 6 | -------------------------------------------------------------------------------- /docs/_scripts/distribution.sh: -------------------------------------------------------------------------------- 1 | # Distributions are created asynchronously. Create one, and specify the publication that will 2 | # be served at the base path specified. 3 | pulp python distribution create --name foo --base-path foo --publication "$PUBLICATION_HREF" 4 | -------------------------------------------------------------------------------- /.pep8speaks.yml: -------------------------------------------------------------------------------- 1 | pycodestyle: 2 | max-line-length: 100 # Default is 79 in PEP8 3 | ignore: # Errors and warnings to ignore 4 | exclude: 5 | - "./docs/*" 6 | - "*/build/*" 7 | - "*/migrations/*" 8 | 9 | # E401: multiple imports on one line 10 | -------------------------------------------------------------------------------- /.github/workflows/scripts/create_release_from_tag.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | set -e 3 | 4 | curl -s -X POST https://api.github.com/repos/$GITHUB_REPOSITORY/releases \ 5 | -H "Authorization: token $RELEASE_TOKEN" \ 6 | -d @- << EOF 7 | { 8 | "tag_name": "$1", 9 | "name": "$1" 10 | } 11 | EOF 12 | -------------------------------------------------------------------------------- /docs/_scripts/artifact.sh: -------------------------------------------------------------------------------- 1 | # Get a Python package or choose your own 2 | curl -O https://fixtures.pulpproject.org/python-pypi/packages/shelf-reader-0.1.tar.gz 3 | export PKG="shelf-reader-0.1.tar.gz" 4 | 5 | # Upload it to Pulp 6 | pulp python content upload --relative-path "$PKG" --file "$PKG" 7 | -------------------------------------------------------------------------------- /docs/_scripts/add_content_repo.sh: -------------------------------------------------------------------------------- 1 | # Add created PythonPackage content to repository 2 | pulp python repository content add --repository foo --filename "shelf-reader-0.1.tar.gz" 3 | 4 | # After the task is complete, it gives us a new repository version 5 | pulp python repository version show --repository foo 6 | -------------------------------------------------------------------------------- /docs/restapi/index.rst: -------------------------------------------------------------------------------- 1 | REST API 2 | ======== 3 | 4 | Pulpcore Reference: `pulpcore REST documentation `_. 5 | 6 | Pulp Python Endpoints 7 | --------------------- 8 | 9 | Pulp Python Reference `pulp-python REST documentation <../restapi.html>`_ 10 | -------------------------------------------------------------------------------- /pulp_python/tests/unit/test_models.py: -------------------------------------------------------------------------------- 1 | from django.test import TestCase 2 | 3 | 4 | class TestNothing(TestCase): 5 | """Test Nothing (placeholder).""" 6 | 7 | def test_nothing_at_all(self): 8 | """Test that the tests are running and that's it.""" 9 | self.assertTrue(True) 10 | -------------------------------------------------------------------------------- /pulp_python/app/__init__.py: -------------------------------------------------------------------------------- 1 | from pulpcore.plugin import PulpPluginAppConfig 2 | 3 | 4 | class PulpPythonPluginAppConfig(PulpPluginAppConfig): 5 | """ 6 | Entry point for pulp_python plugin. 7 | """ 8 | 9 | name = "pulp_python.app" 10 | label = "python" 11 | version = "3.5.0.dev" 12 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE 2 | include requirements.txt 3 | include pyproject.toml 4 | include CHANGES.rst 5 | include COMMITMENT 6 | include COPYRIGHT 7 | include functest_requirements.txt 8 | include test_requirements.txt 9 | include unittest_requirements.txt 10 | include pulp_python/app/webserver_snippets/* 11 | -------------------------------------------------------------------------------- /docs/_scripts/autoupdate.sh: -------------------------------------------------------------------------------- 1 | # This configures the repository to produce new publications when a new version is created 2 | pulp python repository update --name foo --autopublish 3 | # This configures the distribution to be track the latest repository version for a given repository 4 | pulp python distribution update --name foo --repository foo 5 | -------------------------------------------------------------------------------- /docs/_scripts/upload.sh: -------------------------------------------------------------------------------- 1 | source clean.sh 2 | source repo.sh 3 | 4 | # Get a Python package or choose your own 5 | curl -O https://fixtures.pulpproject.org/python-pypi/packages/shelf-reader-0.1.tar.gz 6 | export PKG="shelf-reader-0.1.tar.gz" 7 | 8 | # Upload it to Pulp 9 | pulp python content upload --relative-path "$PKG" --file "$PKG" 10 | -------------------------------------------------------------------------------- /docs/_scripts/sync.sh: -------------------------------------------------------------------------------- 1 | # Using the Remote we just created, we kick off a sync task 2 | pulp python repository sync --name foo --remote bar 3 | 4 | # The sync command will by default wait for the sync to complete 5 | # Use Ctrl+c or the -b option to send the task to the background 6 | 7 | # Show the latest version when sync is done 8 | pulp python repository version show --repository foo 9 | -------------------------------------------------------------------------------- /docs/_scripts/publication.sh: -------------------------------------------------------------------------------- 1 | # Create a new publication specifying the repository_version. 2 | # Alternatively, you can specify just the repository, and Pulp will assume the latest version. 3 | pulp python publication create --repository foo --version 1 4 | 5 | # Publications can only be referenced through their pulp_href 6 | PUBLICATION_HREF=$(pulp python publication list | jq -r .[0].pulp_href) 7 | -------------------------------------------------------------------------------- /pulp_python/app/webserver_snippets/nginx.conf: -------------------------------------------------------------------------------- 1 | location /pypi/ { 2 | proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; 3 | proxy_set_header X-Forwarded-Proto $scheme; 4 | proxy_set_header Host $http_host; 5 | # we don't want nginx trying to do something clever with 6 | # redirects, we set the Host: header above already. 7 | proxy_redirect off; 8 | proxy_pass http://pulp-api; 9 | } 10 | -------------------------------------------------------------------------------- /docs/_static/survey_banner.js: -------------------------------------------------------------------------------- 1 | window.onload = function() { 2 | var elem = document.createElement('div'); 3 | var body = document.getElementsByClassName("rst-content")[0] 4 | var doc = document.getElementsByClassName("document")[0] 5 | elem.className = "admonition important" 6 | elem.id = "pulp-survey-banner" 7 | elem.innerHTML = "

Please take our survey to help us improve Pulp!

"; 8 | body.insertBefore(elem, doc) 9 | } 10 | -------------------------------------------------------------------------------- /.bumpversion.cfg: -------------------------------------------------------------------------------- 1 | [bumpversion] 2 | current_version = 3.5.0.dev 3 | commit = False 4 | tag = False 5 | parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\.(?P[a-z]+))? 6 | serialize = 7 | {major}.{minor}.{patch}.{release} 8 | {major}.{minor}.{patch} 9 | 10 | [bumpversion:part:release] 11 | optional_value = prod 12 | first_value = dev 13 | values = 14 | dev 15 | prod 16 | 17 | [bumpversion:file:./pulp_python/app/__init__.py] 18 | 19 | [bumpversion:file:./setup.py] 20 | 21 | [bumpversion:file:./docs/conf.py] 22 | -------------------------------------------------------------------------------- /docs/tech-preview.rst: -------------------------------------------------------------------------------- 1 | Tech previews 2 | ============= 3 | 4 | The following features are currently being released as part of a tech preview 5 | 6 | * New endpoint “pulp/api/v3/remotes/python/python/from_bandersnatch/” that allows for Python remote creation from a 7 | Bandersnatch config file. 8 | * PyPI’s json API at content endpoint ‘/pypi/{package-name}/json’. Allows for basic Pulp-to-Pulp syncing. 9 | * Fully mirror Python repositories like PyPI. 10 | * ``Twine`` upload packages to indexes at endpoints '/simple` or '/legacy'. 11 | -------------------------------------------------------------------------------- /pulp_python/app/migrations/0006_pythonrepository_autopublish.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.20 on 2021-04-27 20:14 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ('python', '0005_pythonpackagecontent_sha256'), 10 | ] 11 | 12 | operations = [ 13 | migrations.AddField( 14 | model_name='pythonrepository', 15 | name='autopublish', 16 | field=models.BooleanField(default=False), 17 | ), 18 | ] 19 | -------------------------------------------------------------------------------- /pulp_python/app/migrations/0009_pythondistribution_allow_uploads.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.24 on 2021-06-10 14:50 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ('python', '0008_pythonpackagecontent_unique_sha256'), 10 | ] 11 | 12 | operations = [ 13 | migrations.AddField( 14 | model_name='pythondistribution', 15 | name='allow_uploads', 16 | field=models.BooleanField(default=True), 17 | ), 18 | ] 19 | -------------------------------------------------------------------------------- /.github/workflows/scripts/secrets.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | import sys 4 | 5 | secrets = json.loads(sys.argv[1]) 6 | for key, value in secrets.items(): 7 | print(f"Setting {key} ...") 8 | lines = len(value.split("\n")) 9 | if lines > 1: 10 | os.system(f"/bin/bash -c \"echo '{key}<> $GITHUB_ENV\"") 11 | os.system(f"/bin/bash -c \"echo '{value}' >> $GITHUB_ENV\"") 12 | os.system("/bin/bash -c \"echo 'EOF' >> $GITHUB_ENV\"") 13 | else: 14 | os.system(f"/bin/bash -c \"echo '{key}={value}' >> $GITHUB_ENV\"") 15 | -------------------------------------------------------------------------------- /pulp_python/app/migrations/0002_pythonpackagecontent_python_version.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.3 on 2020-08-12 22:25 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ('python', '0001_initial'), 10 | ] 11 | 12 | operations = [ 13 | migrations.AddField( 14 | model_name='pythonpackagecontent', 15 | name='python_version', 16 | field=models.TextField(default=''), 17 | preserve_default=False, 18 | ), 19 | ] 20 | -------------------------------------------------------------------------------- /COPYRIGHT: -------------------------------------------------------------------------------- 1 | Copyright © 2014 Pulp Project developers. 2 | 3 | This software is licensed to you under the GNU General Public 4 | License as published by the Free Software Foundation; either version 5 | 2 of the License (GPLv2) or (at your option) any later version. 6 | There is NO WARRANTY for this software, express or implied, 7 | including the implied warranties of MERCHANTABILITY, 8 | NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should 9 | have received a copy of GPLv2 along with this software; if not, see 10 | http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. 11 | -------------------------------------------------------------------------------- /.github/workflows/scripts/utils.sh: -------------------------------------------------------------------------------- 1 | # This file is meant to be sourced by ci-scripts 2 | 3 | # WARNING: DO NOT EDIT! 4 | # 5 | # This file was generated by plugin_template, and is managed by it. Please use 6 | # './plugin-template --github pulp_python' to update this file. 7 | # 8 | # For more info visit https://github.com/pulp/plugin_template 9 | 10 | PULP_CI_CONTAINER=pulp 11 | 12 | # Run a command 13 | cmd_prefix() { 14 | docker exec "$PULP_CI_CONTAINER" "$@" 15 | } 16 | 17 | # Run a command, and pass STDIN 18 | cmd_stdin_prefix() { 19 | docker exec -i "$PULP_CI_CONTAINER" "$@" 20 | } 21 | -------------------------------------------------------------------------------- /.github/workflows/scripts/push_branch_and_tag_to_github.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | set -e 3 | 4 | BRANCH_NAME=$(echo $GITHUB_REF | sed -rn 's/refs\/heads\/(.*)/\1/p') 5 | 6 | ref_string=$(git show-ref --tags | grep refs/tags/$1) 7 | 8 | SHA=${ref_string:0:40} 9 | 10 | remote_repo=https://pulpbot:${RELEASE_TOKEN}@github.com/${GITHUB_REPOSITORY}.git 11 | 12 | git push "${remote_repo}" $BRANCH_NAME 13 | 14 | curl -s -X POST https://api.github.com/repos/$GITHUB_REPOSITORY/git/refs \ 15 | -H "Authorization: token $RELEASE_TOKEN" \ 16 | -d @- << EOF 17 | { 18 | "ref": "refs/tags/$1", 19 | "sha": "$SHA" 20 | } 21 | EOF 22 | -------------------------------------------------------------------------------- /docs/_scripts/clean.sh: -------------------------------------------------------------------------------- 1 | if [ -n "$(pulp python repository list | grep foo)" ]; then 2 | pulp python repository destroy --name foo 3 | fi 4 | 5 | if [ -n "$(pulp python remote list | grep foo)" ]; then 6 | pulp python remote destroy --name bar 7 | fi 8 | 9 | if [ -n "$(pulp python publication list | grep pulp)" ]; then 10 | pulp python publication destroy --href "$(pulp python publication list | jq .[0].pulp_href)" 11 | fi 12 | 13 | if [ -n "$(pulp python distribution list | grep foo)" ]; then 14 | pulp python distribution destroy --name foo 15 | fi 16 | 17 | pulp orphans delete 18 | pip uninstall -y shelf-reader 19 | -------------------------------------------------------------------------------- /.ci/scripts/check_gettext.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # WARNING: DO NOT EDIT! 4 | # 5 | # This file was generated by plugin_template, and is managed by it. Please use 6 | # './plugin-template --github pulp_python' to update this file. 7 | # 8 | # For more info visit https://github.com/pulp/plugin_template 9 | 10 | # make sure this script runs at the repo root 11 | cd "$(dirname "$(realpath -e "$0")")"/../.. 12 | 13 | set -uv 14 | 15 | # check for imports not from pulpcore.plugin. exclude tests 16 | MATCHES=$(grep -n -r --include \*.py "_(f") 17 | 18 | if [ $? -ne 1 ]; then 19 | printf "\nERROR: Detected mix of f-strings and gettext:\n" 20 | echo "$MATCHES" 21 | exit 1 22 | fi 23 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.towncrier] 2 | package = "pulp_python" 3 | filename = "CHANGES.rst" 4 | directory = "CHANGES/" 5 | title_format = "{version} ({project_date})" 6 | template = "CHANGES/.TEMPLATE.rst" 7 | issue_format = "`#{issue} `_" 8 | 9 | [tool.check-manifest] 10 | ignore = [ 11 | ".bumpversion.cfg", 12 | ".pep8speaks.yml", 13 | "CHANGES/**", 14 | "CONTRIBUTING.rst", 15 | "HISTORY.rst", 16 | "dev_requirements.txt", 17 | "doc_requirements.txt", 18 | "docs/**", 19 | "flake8.cfg", 20 | "template_config.yml", 21 | ".travis/**", 22 | ".travis.yml", 23 | "shelf_reader-0.1-py2-none-any.whl", 24 | ".github/**", 25 | ".ci/**", 26 | ] 27 | -------------------------------------------------------------------------------- /.ci/ansible/smash-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "pulp": { 3 | "auth": [ 4 | "admin", 5 | "password" 6 | ], 7 | "selinux enabled": false, 8 | "version": "3" 9 | }, 10 | "hosts": [ 11 | { 12 | "hostname": "pulp", 13 | "roles": { 14 | "api": { 15 | "port": 80, 16 | "scheme": "http", 17 | "service": "nginx" 18 | }, 19 | "content": { 20 | "port": 80, 21 | "scheme": "http", 22 | "service": "pulp_content_app" 23 | }, 24 | "pulp resource manager": {}, 25 | "pulp workers": {}, 26 | "redis": {}, 27 | "shell": { 28 | "transport": "docker" 29 | } 30 | } 31 | } 32 | ] 33 | } 34 | -------------------------------------------------------------------------------- /.ci/scripts/check_pulpcore_imports.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # WARNING: DO NOT EDIT! 4 | # 5 | # This file was generated by plugin_template, and is managed by it. Please use 6 | # './plugin-template --github pulp_python' to update this file. 7 | # 8 | # For more info visit https://github.com/pulp/plugin_template 9 | 10 | # make sure this script runs at the repo root 11 | cd "$(dirname "$(realpath -e "$0")")"/../.. 12 | 13 | set -uv 14 | 15 | # check for imports not from pulpcore.plugin. exclude tests 16 | MATCHES=$(grep -n -r --include \*.py "from pulpcore.*import" . | grep -v "tests\|plugin") 17 | 18 | if [ $? -ne 1 ]; then 19 | printf "\nERROR: Detected bad imports from pulpcore:\n" 20 | echo "$MATCHES" 21 | printf "\nPlugins should import from pulpcore.plugin." 22 | exit 1 23 | fi 24 | -------------------------------------------------------------------------------- /.github/workflows/scripts/check_commit.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # WARNING: DO NOT EDIT! 4 | # 5 | # This file was generated by plugin_template, and is managed by it. Please use 6 | # './plugin-template --github pulp_python' to update this file. 7 | # 8 | # For more info visit https://github.com/pulp/plugin_template 9 | 10 | # make sure this script runs at the repo root 11 | cd "$(dirname "$(realpath -e "$0")")"/../../.. 12 | 13 | set -euv 14 | 15 | echo ::group::REQUESTS 16 | pip3 install requests 17 | 18 | pip3 install pygithub 19 | 20 | echo ::endgroup:: 21 | 22 | for sha in $(curl $GITHUB_CONTEXT | jq '.[].sha' | sed 's/"//g') 23 | do 24 | python3 .ci/scripts/validate_commit_message.py $sha 25 | VALUE=$? 26 | if [ "$VALUE" -gt 0 ]; then 27 | exit $VALUE 28 | fi 29 | done 30 | -------------------------------------------------------------------------------- /.ci/ansible/filter/repr.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, division, print_function 2 | from packaging.version import parse as parse_version 3 | 4 | __metaclass__ = type 5 | 6 | 7 | ANSIBLE_METADATA = { 8 | "metadata_version": "1.1", 9 | "status": ["preview"], 10 | "supported_by": "community", 11 | } 12 | 13 | 14 | def _repr_filter(value): 15 | return repr(value) 16 | 17 | 18 | def _canonical_semver_filter(value): 19 | return str(parse_version(value)) 20 | 21 | 22 | # ---- Ansible filters ---- 23 | class FilterModule(object): 24 | """Repr filter.""" 25 | 26 | def filters(self): 27 | """Filter associations.""" 28 | return { 29 | "repr": _repr_filter, 30 | "canonical_semver": _canonical_semver_filter, 31 | } 32 | -------------------------------------------------------------------------------- /docs/_templates/restapi.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | REST API for Pulp 3 python Plugin 5 | 6 | 7 | 8 | 9 | 10 | 13 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | -------------------------------------------------------------------------------- /docs/_scripts/base.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | echo "Setting environment variables for default hostname/port for the API and the Content app" 4 | export BASE_ADDR=${BASE_ADDR:-http://localhost:24817} 5 | export CONTENT_ADDR=${CONTENT_ADDR:-http://localhost:24816} 6 | 7 | # Necessary for `django-admin` 8 | export DJANGO_SETTINGS_MODULE=pulpcore.app.settings 9 | 10 | # Install from source 11 | if [ -z "$(pip freeze | grep pulp-cli)" ]; then 12 | echo "Installing Pulp CLI" 13 | git clone https://github.com/pulp/pulp-cli.git 14 | cd pulp-cli 15 | pip install -e . 16 | cd .. 17 | fi 18 | 19 | # Set up CLI config file 20 | mkdir ~/.config/pulp 21 | cat > ~/.config/pulp/settings.toml << EOF 22 | [cli] 23 | base_url = "$BASE_ADDR" # common to be localhost 24 | verify_ssl = false 25 | format = "json" 26 | EOF 27 | -------------------------------------------------------------------------------- /docs/_scripts/quickstart.sh: -------------------------------------------------------------------------------- 1 | # This script will execute the component scripts and ensure that the documented examples 2 | # work as expected. 3 | 4 | # THIS SCRIPT CURRENTLY MUST BE RUN IN A PULPLIFT DEVELOPMENT ENVIRONMENT 5 | # TODO: remove the usage of pulp-devel bash functions so they can be directly modified 6 | # for user environments. 7 | 8 | # From the _scripts directory, run with `source quickstart.sh` (source to preserve the environment 9 | # variables) 10 | export PLUGIN_SOURCE="../../" 11 | set -e 12 | 13 | source base.sh 14 | source clean.sh 15 | 16 | source repo.sh 17 | source remote.sh 18 | source sync.sh 19 | 20 | source publication.sh 21 | source distribution.sh 22 | source autoupdate.sh 23 | source pip.sh 24 | 25 | source upload.sh 26 | source add_content_repo.sh 27 | 28 | source index.sh 29 | source twine.sh 30 | -------------------------------------------------------------------------------- /pulp_python/app/migrations/0007_pythonpackagecontent_mv-2-1.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.23 on 2021-06-04 15:50 2 | 3 | import django.contrib.postgres.fields.jsonb 4 | from django.db import migrations, models 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ('python', '0006_pythonrepository_autopublish'), 11 | ] 12 | 13 | operations = [ 14 | migrations.AddField( 15 | model_name='pythonpackagecontent', 16 | name='description_content_type', 17 | field=models.TextField(default=''), 18 | preserve_default=False, 19 | ), 20 | migrations.AddField( 21 | model_name='pythonpackagecontent', 22 | name='project_urls', 23 | field=django.contrib.postgres.fields.jsonb.JSONField(default=dict), 24 | ), 25 | ] 26 | -------------------------------------------------------------------------------- /flake8.cfg: -------------------------------------------------------------------------------- 1 | [flake8] 2 | exclude = ./docs/*,*/migrations/*,.github/*,.ci/* 3 | ignore = Q000,Q003,D100,D104,D106,D200,D205,D400,D401,D402,D413 4 | max-line-length = 100 5 | 6 | # Flake8-quotes extension codes 7 | # ----------------------------- 8 | # Q000: double or single quotes only, default is double (don't want to enforce this) 9 | 10 | # Flake8-docstring extension codes 11 | # -------------------------------- 12 | # D100: missing docstring in public module 13 | # D104: missing docstring in public package 14 | # D106: missing docstring in public nested class (complains about "class Meta:" and documenting those is silly) 15 | # D200: one-line docstring should fit on one line with quotes 16 | # D401: first line should be imperative (nitpicky) 17 | # D402: first line should not be the function’s “signature” (false positives) 18 | # D413: missing blank line after last section 19 | -------------------------------------------------------------------------------- /.ci/scripts/schema.py: -------------------------------------------------------------------------------- 1 | """ 2 | Customizing OpenAPI validation. 3 | 4 | OpenAPI requires paths to start with slashes: 5 | https://spec.openapis.org/oas/v3.0.3#patterned-fields 6 | 7 | But some pulp paths start with curly brackets e.g. {artifact_href} 8 | This script modifies drf-spectacular schema validation to accept slashes and curly brackets. 9 | """ 10 | import json 11 | from drf_spectacular.validation import JSON_SCHEMA_SPEC_PATH 12 | 13 | with open(JSON_SCHEMA_SPEC_PATH) as fh: 14 | openapi3_schema_spec = json.load(fh) 15 | 16 | properties = openapi3_schema_spec["definitions"]["Paths"]["patternProperties"] 17 | # Making OpenAPI validation to accept paths starting with / and { 18 | if "^\\/|{" not in properties: 19 | properties["^\\/|{"] = properties["^\\/"] 20 | del properties["^\\/"] 21 | 22 | with open(JSON_SCHEMA_SPEC_PATH, "w") as fh: 23 | json.dump(openapi3_schema_spec, fh) 24 | -------------------------------------------------------------------------------- /.ci/scripts/update_redmine.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # WARNING: DO NOT EDIT! 4 | # 5 | # This file was generated by plugin_template, and is managed by it. Please use 6 | # './plugin-template --github pulp_python' to update this file. 7 | # 8 | # For more info visit https://github.com/pulp/plugin_template 9 | 10 | # make sure this script runs at the repo root 11 | cd "$(dirname "$(realpath -e "$0")")"/../.. 12 | 13 | set -euv 14 | 15 | export COMMIT_MSG=$(git log --format=%B --no-merges -1) 16 | export RELEASE=$(echo $COMMIT_MSG | awk '{print $2}') 17 | export MILESTONE_URL=$(echo $COMMIT_MSG | grep -o "Redmine Milestone: .*" | awk '{print $3}') 18 | export REDMINE_QUERY_URL=$(echo $COMMIT_MSG | grep -o "Redmine Query: .*" | awk '{print $3}') 19 | 20 | echo "Releasing $RELEASE" 21 | echo "Milestone URL: $MILESTONE_URL" 22 | echo "Query: $REDMINE_QUERY_URL" 23 | 24 | pip install python-redmine httpie 25 | 26 | python3 .ci/scripts/redmine.py $REDMINE_QUERY_URL $MILESTONE_URL $RELEASE 27 | -------------------------------------------------------------------------------- /CHANGES/.TEMPLATE.rst: -------------------------------------------------------------------------------- 1 | 2 | {# TOWNCRIER TEMPLATE #} 3 | {% for section, _ in sections.items() %} 4 | {% set underline = underlines[0] %}{% if section %}{{section}} 5 | {{ underline * section|length }}{% set underline = underlines[1] %} 6 | 7 | {% endif %} 8 | 9 | {% if sections[section] %} 10 | {% for category, val in definitions.items() if category in sections[section]%} 11 | {{ definitions[category]['name'] }} 12 | {{ underline * definitions[category]['name']|length }} 13 | 14 | {% if definitions[category]['showcontent'] %} 15 | {% for text, values in sections[section][category].items() %} 16 | - {{ text }} 17 | {{ values|join(',\n ') }} 18 | {% endfor %} 19 | 20 | {% else %} 21 | - {{ sections[section][category]['']|join(', ') }} 22 | 23 | {% endif %} 24 | {% if sections[section][category]|length == 0 %} 25 | No significant changes. 26 | 27 | {% else %} 28 | {% endif %} 29 | 30 | {% endfor %} 31 | {% else %} 32 | No significant changes. 33 | 34 | 35 | {% endif %} 36 | {% endfor %} 37 | ---- 38 | -------------------------------------------------------------------------------- /.ci/ansible/Containerfile.j2: -------------------------------------------------------------------------------- 1 | FROM {{ ci_base | default("pulp/pulp-ci-centos:latest") }} 2 | 3 | # Add source directories to container 4 | {% for item in plugins %} 5 | {% if item.source.startswith("./") %} 6 | ADD {{ item.source }} {{ item.source }} 7 | {% endif %} 8 | {% endfor %} 9 | 10 | # Install python packages 11 | # Hacking botocore (https://github.com/boto/botocore/pull/1990) 12 | 13 | RUN pip3 install \ 14 | {%- if s3_test | default(false) -%} 15 | {{ " " }}django-storages[boto3] git+https://github.com/fabricio-aguiar/botocore.git@fix-100-continue 16 | {%- endif -%} 17 | {%- for item in plugins -%} 18 | {%- if item.name == "pulp-certguard" -%} 19 | {{ " " }}python-dateutil rhsm 20 | {%- endif -%} 21 | {{ " " }}"{{ item.source }}" 22 | {%- endfor %} 23 | 24 | RUN mkdir -p /etc/nginx/pulp/ 25 | {% for item in plugins %} 26 | RUN ln /usr/local/lib/python3.8/site-packages/{{ item.name }}/app/webserver_snippets/nginx.conf /etc/nginx/pulp/{{ item.name }}.conf || true 27 | {% endfor %} 28 | 29 | ENTRYPOINT ["/init"] 30 | -------------------------------------------------------------------------------- /.ci/ansible/settings.py.j2: -------------------------------------------------------------------------------- 1 | CONTENT_ORIGIN = "http://pulp:80" 2 | ANSIBLE_API_HOSTNAME = "http://pulp:80" 3 | ANSIBLE_CONTENT_HOSTNAME = "http://pulp:80/pulp/content" 4 | PRIVATE_KEY_PATH = "/etc/pulp/certs/token_private_key.pem" 5 | PUBLIC_KEY_PATH = "/etc/pulp/certs/token_public_key.pem" 6 | TOKEN_SERVER = "http://pulp:80/token/" 7 | TOKEN_SIGNATURE_ALGORITHM = "ES256" 8 | 9 | {% if pulp_settings %} 10 | {% for key, value in pulp_settings.items() %} 11 | {{ key | upper }} = {{ value | repr }} 12 | {% endfor %} 13 | {% endif %} 14 | 15 | {% if s3_test | default(false) %} 16 | AWS_ACCESS_KEY_ID = "{{ minio_access_key }}" 17 | AWS_SECRET_ACCESS_KEY = "{{ minio_secret_key }}" 18 | AWS_S3_REGION_NAME = "eu-central-1" 19 | AWS_S3_ADDRESSING_STYLE = "path" 20 | S3_USE_SIGV4 = True 21 | AWS_S3_SIGNATURE_VERSION = "s3v4" 22 | AWS_STORAGE_BUCKET_NAME = "pulp3" 23 | AWS_S3_ENDPOINT_URL = "http://minio:9000" 24 | DEFAULT_FILE_STORAGE = "storages.backends.s3boto3.S3Boto3Storage" 25 | AWS_DEFAULT_ACL = "@none None" 26 | {% endif %} 27 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | 5 | # C extensions 6 | *.so 7 | 8 | # Distribution / packaging 9 | .Python 10 | env/ 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | lib/ 17 | lib64/ 18 | parts/ 19 | sdist/ 20 | var/ 21 | *.egg-info/ 22 | .installed.cfg 23 | *.egg 24 | pip-wheel-metadata 25 | 26 | # PyInstaller 27 | # Usually these files are written by a python script from a template 28 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 29 | *.manifest 30 | 31 | # Installer logs 32 | pip-log.txt 33 | pip-delete-this-directory.txt 34 | 35 | # Unit test / coverage reports 36 | htmlcov/ 37 | .tox/ 38 | .coverage 39 | .cache 40 | nosetests.xml 41 | coverage.xml 42 | cover/ 43 | .pytest_cache/ 44 | 45 | # Translations 46 | *.mo 47 | *.pot 48 | 49 | # Django stuff: 50 | *.log 51 | 52 | # Sphinx documentation 53 | docs/_build/ 54 | docs/_static/api.json 55 | 56 | # PyBuilder 57 | target/ 58 | 59 | # Ninja IDE 60 | *.nja 61 | 62 | # PyCharm 63 | .idea 64 | -------------------------------------------------------------------------------- /.github/workflows/scripts/publish_plugin_pypi.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # WARNING: DO NOT EDIT! 4 | # 5 | # This file was generated by plugin_template, and is managed by it. Please use 6 | # './plugin-template --github pulp_python' to update this file. 7 | # 8 | # For more info visit https://github.com/pulp/plugin_template 9 | 10 | # make sure this script runs at the repo root 11 | cd "$(dirname "$(realpath -e "$0")")"/../../.. 12 | 13 | set -euv 14 | 15 | export PULP_URL="${PULP_URL:-http://pulp}" 16 | 17 | export response=$(curl --write-out %{http_code} --silent --output /dev/null https://pypi.org/project/pulp-python/$1/) 18 | if [ "$response" == "200" ]; 19 | then 20 | echo "pulp_python $1 has already been released. Skipping." 21 | exit 22 | fi 23 | 24 | pip install twine 25 | 26 | twine check dist/pulp_python-$1-py3-none-any.whl || exit 1 27 | twine check dist/pulp-python-$1.tar.gz || exit 1 28 | twine upload dist/pulp_python-$1-py3-none-any.whl -u pulp -p $PYPI_PASSWORD 29 | twine upload dist/pulp-python-$1.tar.gz -u pulp -p $PYPI_PASSWORD 30 | 31 | exit $? 32 | -------------------------------------------------------------------------------- /pulp_python/app/migrations/0008_pythonpackagecontent_unique_sha256.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.23 on 2021-06-10 01:57 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ('python', '0007_pythonpackagecontent_mv-2-1'), 10 | ] 11 | 12 | operations = [ 13 | migrations.AlterField( 14 | model_name='pythonpackagecontent', 15 | name='filename', 16 | field=models.TextField(db_index=True), 17 | ), 18 | migrations.AlterField( 19 | model_name='pythonpackagecontent', 20 | name='sha256', 21 | field=models.CharField(db_index=True, max_length=64, unique=True), 22 | ), 23 | migrations.SeparateDatabaseAndState( 24 | state_operations=[ 25 | migrations.AlterUniqueTogether( 26 | name='pythonpackagecontent', 27 | unique_together={('sha256',)}, 28 | ), 29 | ], 30 | ), 31 | ] 32 | -------------------------------------------------------------------------------- /docs/workflows/index.rst: -------------------------------------------------------------------------------- 1 | .. _workflows-index: 2 | 3 | Workflows 4 | ========= 5 | 6 | If you have not yet installed the `python` plugin on your Pulp installation, please follow our 7 | :doc:`../installation`. These documents will assume you have the environment installed and 8 | ready to go. 9 | 10 | The example workflows here use the Pulp CLI. Get and setup the Pulp CLI from PyPI with the following 11 | commands. For more information about setting up the Pulp CLI please read the `installation and configuration 12 | doc page `_. 13 | 14 | .. code-block:: bash 15 | 16 | pip install pulp-cli[pygments] # For colored output 17 | pulp config create -e 18 | 19 | If you configured the ``admin`` user with a different password, adjust the configuration 20 | accordingly. If you prefer to specify the username and password with each request, please see 21 | Pulp CLI documentation on how to do that. 22 | 23 | 24 | .. toctree:: 25 | :maxdepth: 2 26 | 27 | pypi 28 | sync 29 | upload 30 | publish 31 | -------------------------------------------------------------------------------- /pulp_python/app/urls.py: -------------------------------------------------------------------------------- 1 | import socket 2 | from django.urls import path 3 | 4 | from pulp_python.app.pypi.views import SimpleView, MetadataView, PyPIView, UploadView 5 | 6 | PYPI_API_URL = 'pypi//' 7 | PYPI_API_HOSTNAME = 'https://' + socket.getfqdn() 8 | # TODO: Implement remaining PyPI endpoints 9 | # path("project/", PackageProject.as_view()), # Endpoints to nicely see contents of index 10 | # path("search/", PackageSearch.as_view()), 11 | 12 | urlpatterns = [ 13 | path(PYPI_API_URL + "legacy/", UploadView.as_view({"post": "create"}), name="upload"), 14 | path( 15 | PYPI_API_URL + "pypi//", 16 | MetadataView.as_view({"get": "retrieve"}), 17 | name="pypi-metadata" 18 | ), 19 | path( 20 | PYPI_API_URL + "simple//", 21 | SimpleView.as_view({"get": "retrieve"}), 22 | name="simple-package-detail" 23 | ), 24 | path( 25 | PYPI_API_URL + 'simple/', 26 | SimpleView.as_view({"get": "list", "post": "create"}), 27 | name="simple-detail" 28 | ), 29 | path(PYPI_API_URL, PyPIView.as_view({"get": "retrieve"}), name="pypi-detail"), 30 | ] 31 | -------------------------------------------------------------------------------- /.ci/ansible/build_container.yaml: -------------------------------------------------------------------------------- 1 | # Ansible playbook to create the pulp service containers image 2 | --- 3 | - hosts: localhost 4 | gather_facts: false 5 | vars_files: 6 | - vars/main.yaml 7 | tasks: 8 | - name: "Generate Containerfile from template" 9 | template: 10 | src: Containerfile.j2 11 | dest: Containerfile 12 | 13 | - name: "Build pulp image" 14 | # We build from the ../.. (parent dir of pulpcore git repo) Docker build 15 | # "context" so that repos like pulp-smash are accessible to Docker 16 | # build. So that PR branches can be used via relative paths. 17 | # 18 | # We default to using the docker build / podman buildah cache, for 19 | # 1-off-builds and CI purposes (which has no cache across CI runs.) 20 | # Run build.yaml with -e cache=false if your builds are using outdated 21 | # layers. 22 | command: "docker build --network host --no-cache={{ not cache | default(true) | bool }} -t {{ image.name }}:{{ image.tag }} -f {{ playbook_dir }}/Containerfile ../../.." 23 | 24 | - name: "Clean image cache" 25 | docker_prune: 26 | images : true 27 | ... 28 | -------------------------------------------------------------------------------- /.github/workflows/scripts/publish_client_gem.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # WARNING: DO NOT EDIT! 4 | # 5 | # This file was generated by plugin_template, and is managed by it. Please use 6 | # './plugin-template --github pulp_python' to update this file. 7 | # 8 | # For more info visit https://github.com/pulp/plugin_template 9 | 10 | set -euv 11 | 12 | # make sure this script runs at the repo root 13 | cd "$(dirname "$(realpath -e "$0")")"/../../.. 14 | 15 | 16 | mkdir ~/.gem || true 17 | touch ~/.gem/credentials 18 | echo "--- 19 | :rubygems_api_key: $RUBYGEMS_API_KEY" > ~/.gem/credentials 20 | sudo chmod 600 ~/.gem/credentials 21 | 22 | export VERSION=$(ls pulp_python_client* | sed -rn 's/pulp_python_client-(.*)\.gem/\1/p') 23 | 24 | if [[ -z "$VERSION" ]]; then 25 | echo "No client package found." 26 | exit 27 | fi 28 | 29 | export response=$(curl --write-out %{http_code} --silent --output /dev/null https://rubygems.org/gems/pulp_python_client/versions/$VERSION) 30 | 31 | if [ "$response" == "200" ]; 32 | then 33 | echo "pulp_python client $VERSION has already been released. Skipping." 34 | exit 35 | fi 36 | 37 | GEM_FILE="$(ls pulp_python_client-*)" 38 | gem push ${GEM_FILE} 39 | -------------------------------------------------------------------------------- /HISTORY.rst: -------------------------------------------------------------------------------- 1 | 3.0.0b6 2 | ======= 3 | 4 | * See all changes `here `_. 5 | 6 | * Adds support for `pulpcore 3.0.0.rc2 `_. 7 | 8 | Changes urls for distributions and publications 9 | 10 | * Adds lazy sync 11 | 12 | * Docs replace snippets with testable scripts 13 | 14 | 3.0.0b5 15 | ======= 16 | 17 | * Fix relative_path to allow pip install 18 | 19 | 3.0.0b4 20 | ======= 21 | 22 | * Adds support for `pulpcore 3.0.0.rc1 `_. 23 | 24 | * Adds excludes support (aka 'blacklist') 25 | 26 | Renames the "projects" field on the remote to "includes". 27 | 28 | Adds a new "excludes" field to the remote which behaves like "includes", except that any specified 29 | releasees or digests are not synced, even if an include specifier matches them. 30 | 31 | Also adds a 'prereleases' field to the remote, which toggles whether prerelease versions should be 32 | synced. This mirrors the 'prereleases' flag that packaging.specifiers.SpecifierSet provides. 33 | 34 | * Removes Python 3.5 support 35 | -------------------------------------------------------------------------------- /.github/workflows/scripts/publish_docs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # WARNING: DO NOT EDIT! 4 | # 5 | # This file was generated by plugin_template, and is managed by it. Please use 6 | # './plugin-template --github pulp_python' to update this file. 7 | # 8 | # For more info visit https://github.com/pulp/plugin_template 9 | 10 | set -euv 11 | 12 | # make sure this script runs at the repo root 13 | cd "$(dirname "$(realpath -e "$0")")"/../../.. 14 | 15 | mkdir ~/.ssh 16 | echo "$PULP_DOCS_KEY" > ~/.ssh/pulp-infra 17 | chmod 600 ~/.ssh/pulp-infra 18 | 19 | echo "docs.pulpproject.org,8.43.85.236 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGXG+8vjSQvnAkq33i0XWgpSrbco3rRqNZr0SfVeiqFI7RN/VznwXMioDDhc+hQtgVhd6TYBOrV07IMcKj+FAzg=" >> /home/runner/.ssh/known_hosts 20 | chmod 644 /home/runner/.ssh/known_hosts 21 | 22 | pip3 install -r doc_requirements.txt 23 | 24 | export PYTHONUNBUFFERED=1 25 | export DJANGO_SETTINGS_MODULE=pulpcore.app.settings 26 | export PULP_SETTINGS=$PWD/.ci/ansible/settings/settings.py 27 | export WORKSPACE=$PWD 28 | 29 | eval "$(ssh-agent -s)" #start the ssh agent 30 | ssh-add ~/.ssh/pulp-infra 31 | 32 | python3 .github/workflows/scripts/docs-publisher.py --build-type $1 --branch $2 33 | -------------------------------------------------------------------------------- /.github/workflows/scripts/publish_client_pypi.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # WARNING: DO NOT EDIT! 4 | # 5 | # This file was generated by plugin_template, and is managed by it. Please use 6 | # './plugin-template --github pulp_python' to update this file. 7 | # 8 | # For more info visit https://github.com/pulp/plugin_template 9 | 10 | set -euv 11 | 12 | # make sure this script runs at the repo root 13 | cd "$(dirname "$(realpath -e "$0")")"/../../.. 14 | 15 | pip install twine 16 | 17 | export VERSION=$(ls dist | sed -rn 's/pulp_python-client-(.*)\.tar.gz/\1/p') 18 | 19 | if [[ -z "$VERSION" ]]; then 20 | echo "No client package found." 21 | exit 22 | fi 23 | 24 | export response=$(curl --write-out %{http_code} --silent --output /dev/null https://pypi.org/project/pulp-python-client/$VERSION/) 25 | 26 | if [ "$response" == "200" ]; 27 | then 28 | echo "pulp_python client $VERSION has already been released. Skipping." 29 | exit 30 | fi 31 | 32 | twine check dist/pulp_python_client-$VERSION-py3-none-any.whl || exit 1 33 | twine check dist/pulp_python-client-$VERSION.tar.gz || exit 1 34 | twine upload dist/pulp_python_client-$VERSION-py3-none-any.whl -u pulp -p $PYPI_PASSWORD 35 | twine upload dist/pulp_python-client-$VERSION.tar.gz -u pulp -p $PYPI_PASSWORD 36 | 37 | exit $? 38 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from setuptools import setup, find_packages 4 | 5 | with open("requirements.txt") as requirements: 6 | requirements = requirements.readlines() 7 | 8 | with open("README.md") as f: 9 | long_description = f.read() 10 | 11 | setup( 12 | name="pulp-python", 13 | version="3.5.0.dev", 14 | description="pulp-python plugin for the Pulp Project", 15 | long_description=long_description, 16 | long_description_content_type="text/markdown", 17 | license="GPLv2+", 18 | python_requires=">=3.8", 19 | author="Pulp Project Developers", 20 | author_email="pulp-list@redhat.com", 21 | url="https://www.pulpproject.org", 22 | install_requires=requirements, 23 | include_package_data=True, 24 | packages=find_packages(exclude=["tests"]), 25 | classifiers=( 26 | "License :: OSI Approved :: GNU General Public License v2 or later (GPLv2+)", 27 | "Operating System :: POSIX :: Linux", 28 | "Development Status :: 5 - Production/Stable", 29 | "Framework :: Django", 30 | "Programming Language :: Python", 31 | "Programming Language :: Python :: 3", 32 | "Programming Language :: Python :: 3.8", 33 | "Programming Language :: Python :: 3.9", 34 | ), 35 | entry_points={"pulpcore.plugin": ["pulp_python = pulp_python:default_app_config", ]}, 36 | ) 37 | -------------------------------------------------------------------------------- /.ci/scripts/redmine.py: -------------------------------------------------------------------------------- 1 | # WARNING: DO NOT EDIT! 2 | # 3 | # This file was generated by plugin_template, and is managed by it. Please use 4 | # './plugin-template --github pulp_python' to update this file. 5 | # 6 | # For more info visit https://github.com/pulp/plugin_template 7 | 8 | import os 9 | import sys 10 | 11 | from redminelib import Redmine 12 | 13 | REDMINE_API_KEY = os.environ["REDMINE_API_KEY"] 14 | REDMINE_QUERY_URL = sys.argv[1] 15 | MILESTONE_URL = sys.argv[2] 16 | RELEASE = sys.argv[3] 17 | CLOSED_CURRENTRELEASE = 11 18 | 19 | redmine = Redmine(REDMINE_QUERY_URL.split("issues")[0], key=REDMINE_API_KEY) 20 | query_issues = REDMINE_QUERY_URL.split("=")[-1].split(",") 21 | milestone_name = redmine.version.get(MILESTONE_URL.split("/")[-1].split(".")[0]).name 22 | if milestone_name != RELEASE: 23 | raise RuntimeError(f"Milestone name, '{milestone_name}', does not match version, '{RELEASE}'.") 24 | 25 | to_update = [] 26 | for issue in query_issues: 27 | status = redmine.issue.get(int(issue)).status.name 28 | if "CLOSE" not in status and status != "MODIFIED": 29 | raise ValueError("One or more issues are not MODIFIED") 30 | if status == "MODIFIED": # Removing the already closed 31 | to_update.append(int(issue)) 32 | 33 | for issue in to_update: 34 | print(f"Closing #{issue}") 35 | redmine.issue.update(issue, status_id=CLOSED_CURRENTRELEASE) 36 | -------------------------------------------------------------------------------- /.github/workflows/scripts/before_script.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # WARNING: DO NOT EDIT! 4 | # 5 | # This file was generated by plugin_template, and is managed by it. Please use 6 | # './plugin-template --github pulp_python' to update this file. 7 | # 8 | # For more info visit https://github.com/pulp/plugin_template 9 | 10 | # make sure this script runs at the repo root 11 | cd "$(dirname "$(realpath -e "$0")")"/../../.. 12 | 13 | set -euv 14 | 15 | source .github/workflows/scripts/utils.sh 16 | 17 | export PRE_BEFORE_SCRIPT=$PWD/.github/workflows/scripts/pre_before_script.sh 18 | export POST_BEFORE_SCRIPT=$PWD/.github/workflows/scripts/post_before_script.sh 19 | 20 | if [[ -f $PRE_BEFORE_SCRIPT ]]; then 21 | source $PRE_BEFORE_SCRIPT 22 | fi 23 | 24 | # Developers should be able to reproduce the containers with this config 25 | echo "CI vars:" 26 | tail -v -n +1 .ci/ansible/vars/main.yaml 27 | 28 | # Developers often want to know the final pulp config 29 | echo "PULP CONFIG:" 30 | tail -v -n +1 .ci/ansible/settings/settings.* ~/.config/pulp_smash/settings.json 31 | 32 | if [[ "$TEST" == 'pulp' || "$TEST" == 'performance' || "$TEST" == 'upgrade' || "$TEST" == 's3' || "$TEST" == "plugin-from-pypi" ]]; then 33 | # Many functional tests require these 34 | cmd_prefix dnf install -yq lsof which dnf-plugins-core 35 | fi 36 | 37 | if [[ -f $POST_BEFORE_SCRIPT ]]; then 38 | source $POST_BEFORE_SCRIPT 39 | fi 40 | -------------------------------------------------------------------------------- /pulp_python/app/migrations/0003_new_sync_filters.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.19 on 2021-03-26 13:31 2 | 3 | import django.contrib.postgres.fields 4 | from django.db import migrations, models 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ('python', '0002_pythonpackagecontent_python_version'), 11 | ] 12 | 13 | operations = [ 14 | migrations.AddField( 15 | model_name='pythonremote', 16 | name='exclude_platforms', 17 | field=django.contrib.postgres.fields.ArrayField(base_field=models.CharField(blank=True, max_length=10), choices=[('windows', 'windows'), ('macos', 'macos'), ('freebsd', 'freebsd'), ('linux', 'linux')], default=list, size=None), 18 | ), 19 | migrations.AddField( 20 | model_name='pythonremote', 21 | name='keep_latest_packages', 22 | field=models.IntegerField(default=0), 23 | ), 24 | migrations.AddField( 25 | model_name='pythonremote', 26 | name='package_types', 27 | field=django.contrib.postgres.fields.ArrayField(base_field=models.CharField(blank=True, max_length=15), choices=[('bdist_dmg', 'bdist_dmg'), ('bdist_dumb', 'bdist_dumb'), ('bdist_egg', 'bdist_egg'), ('bdist_msi', 'bdist_msi'), ('bdist_rpm', 'bdist_rpm'), ('bdist_wheel', 'bdist_wheel'), ('bdist_wininst', 'bdist_wininst'), ('sdist', 'sdist')], default=list, size=None), 28 | ), 29 | ] 30 | -------------------------------------------------------------------------------- /template_config.yml: -------------------------------------------------------------------------------- 1 | # This config represents the latest values used when running the plugin-template. Any settings that 2 | # were not present before running plugin-template have been added with their default values. 3 | 4 | additional_plugins: [] 5 | additional_repos: [] 6 | black: false 7 | check_commit_message: true 8 | check_gettext: true 9 | check_manifest: true 10 | check_openapi_schema: true 11 | check_stray_pulpcore_imports: true 12 | cherry_pick_automation: false 13 | coverage: false 14 | deploy_client_to_pypi: true 15 | deploy_client_to_rubygems: true 16 | deploy_daily_client_to_pypi: true 17 | deploy_daily_client_to_rubygems: true 18 | deploy_to_pypi: true 19 | docker_fixtures: false 20 | docs_test: true 21 | issue_tracker: github 22 | plugin_app_label: python 23 | plugin_camel: PulpPython 24 | plugin_camel_short: Python 25 | plugin_caps: PULP_PYTHON 26 | plugin_caps_short: PYTHON 27 | plugin_dash: pulp-python 28 | plugin_dash_short: python 29 | plugin_default_branch: master 30 | plugin_name: pulp_python 31 | plugin_snake: pulp_python 32 | publish_docs_to_pulpprojectdotorg: true 33 | pulp_settings: null 34 | pulpcore_branch: master 35 | pulpcore_pip_version_specifier: null 36 | pulpprojectdotorg_key_id: null 37 | pydocstyle: true 38 | pypi_username: pulp 39 | redmine_project: null 40 | release_user: pulpbot 41 | stable_branch: null 42 | test_bindings: false 43 | test_cli: false 44 | test_fips_nightly: false 45 | test_performance: false 46 | test_released_plugin_with_next_pulpcore_release: false 47 | test_s3: false 48 | travis_addtl_services: [] 49 | travis_notifications: None 50 | update_redmine: false 51 | upgrade_range: [] 52 | 53 | -------------------------------------------------------------------------------- /.github/workflows/scripts/install_python_client.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # WARNING: DO NOT EDIT! 4 | # 5 | # This file was generated by plugin_template, and is managed by it. Please use 6 | # './plugin-template --github pulp_python' to update this file. 7 | # 8 | # For more info visit https://github.com/pulp/plugin_template 9 | 10 | set -euv 11 | 12 | export PULP_URL="${PULP_URL:-http://pulp}" 13 | 14 | # make sure this script runs at the repo root 15 | cd "$(dirname "$(realpath -e "$0")")"/../../.. 16 | 17 | pip install twine wheel 18 | 19 | export REPORTED_VERSION=$(http pulp/pulp/api/v3/status/ | jq --arg plugin python --arg legacy_plugin pulp_python -r '.versions[] | select(.component == $plugin or .component == $legacy_plugin) | .version') 20 | export DESCRIPTION="$(git describe --all --exact-match `git rev-parse HEAD`)" 21 | if [[ $DESCRIPTION == 'tags/'$REPORTED_VERSION ]]; then 22 | export VERSION=${REPORTED_VERSION} 23 | else 24 | export EPOCH="$(date +%s)" 25 | export VERSION=${REPORTED_VERSION}${EPOCH} 26 | fi 27 | 28 | export response=$(curl --write-out %{http_code} --silent --output /dev/null https://pypi.org/project/pulp-python-client/$VERSION/) 29 | 30 | if [ "$response" == "200" ]; 31 | then 32 | echo "pulp_python client $VERSION has already been released. Installing from PyPI." 33 | pip install pulp-python-client==$VERSION 34 | mkdir -p dist 35 | tar cvf python-client.tar ./dist 36 | exit 37 | fi 38 | 39 | cd ../pulp-openapi-generator 40 | 41 | ./generate.sh pulp_python python $VERSION 42 | cd pulp_python-client 43 | python setup.py sdist bdist_wheel --python-tag py3 44 | pip install dist/pulp_python_client-$VERSION-py3-none-any.whl 45 | tar cvf ../../pulp_python/python-client.tar ./dist 46 | exit $? 47 | -------------------------------------------------------------------------------- /.github/workflows/scripts/install_ruby_client.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # WARNING: DO NOT EDIT! 4 | # 5 | # This file was generated by plugin_template, and is managed by it. Please use 6 | # './plugin-template --github pulp_python' to update this file. 7 | # 8 | # For more info visit https://github.com/pulp/plugin_template 9 | 10 | set -euv 11 | 12 | # make sure this script runs at the repo root 13 | cd "$(dirname "$(realpath -e "$0")")"/../../.. 14 | 15 | export PULP_URL="${PULP_URL:-http://pulp}" 16 | 17 | export REPORTED_VERSION=$(http pulp/pulp/api/v3/status/ | jq --arg plugin python --arg legacy_plugin pulp_python -r '.versions[] | select(.component == $plugin or .component == $legacy_plugin) | .version') 18 | export DESCRIPTION="$(git describe --all --exact-match `git rev-parse HEAD`)" 19 | if [[ $DESCRIPTION == 'tags/'$REPORTED_VERSION ]]; then 20 | export VERSION=${REPORTED_VERSION} 21 | else 22 | export EPOCH="$(date +%s)" 23 | export VERSION=${REPORTED_VERSION}${EPOCH} 24 | fi 25 | 26 | export response=$(curl --write-out %{http_code} --silent --output /dev/null https://rubygems.org/gems/pulp_python_client/versions/$VERSION) 27 | 28 | if [ "$response" == "200" ]; 29 | then 30 | echo "pulp_python client $VERSION has already been released. Installing from RubyGems.org." 31 | gem install pulp_python_client -v $VERSION 32 | touch pulp_python_client-$VERSION.gem 33 | tar cvf ruby-client.tar ./pulp_python_client-$VERSION.gem 34 | exit 35 | fi 36 | 37 | cd ../pulp-openapi-generator 38 | 39 | ./generate.sh pulp_python ruby $VERSION 40 | cd pulp_python-client 41 | gem build pulp_python_client 42 | gem install --both ./pulp_python_client-$VERSION.gem 43 | tar cvf ../../pulp_python/ruby-client.tar ./pulp_python_client-$VERSION.gem 44 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # pulp_python 2 | 3 | ![Pulp CI](https://github.com/pulp/pulp_python/actions/workflows/ci.yml/badge.svg?branch=master) 4 | 5 | ![Pulp Nightly CI/CD](https://github.com/pulp/pulp_python/actions/workflows/nightly.yml/badge.svg) 6 | 7 | A Pulp plugin to support hosting your own pip compatible Python packages. 8 | 9 | For more information, please see the [documentation](https://pulp-python.readthedocs.io/en/latest/) or the 10 | [Pulp project page](https://pulpproject.org). 11 | 12 | ## Replit Fork 13 | **For this fork in particular, the documentation should be accessed in the `docs` directory directly as we 14 | have modified it.** 15 | 16 | This plugin should never be installed manually and instead installed as part of the ansible playbook for 17 | pulp in goval. It may happen, however, that you will need to make modifications or test things inside of a virtual 18 | machine configured with ansible. In these cases, here are a few instructions on how to get going. 19 | 20 | ### Updating an ansible VM with pip 21 | If you have uploaded you work, you can update the version of `pulp_python` installed on your VM by 22 | installing wit pip and git. Type `/usr/local/lib/pulp/bin/pip install git+https://github.com/replit/pulp_python@your_branch` 23 | in the console in your VM, and it should install the plugin. Always restart the workers and content 24 | service by typing `sudo systemctl restart pulpcore-worker@* && sudo systemctl restart pulpcore-content`. 25 | 26 | ### Updating the code directly in the VM 27 | If you need to update the code for pulp_python directly in the VM, you can do so by modifying the 28 | plugin code stored in `/usr/local/lib/pulp/lib/python3.8/site-packages/pulp_python`. Always deleted all 29 | the `__cache__` directories saved there, then restart the services to see your changes. 30 | -------------------------------------------------------------------------------- /pulp_python/app/migrations/0005_pythonpackagecontent_sha256.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.20 on 2021-04-26 16:28 2 | 3 | from django.db import migrations, models, transaction 4 | 5 | 6 | def add_sha256_to_current_models(apps, schema_editor): 7 | """Adds the sha256 to current PythonPackageContent models.""" 8 | PythonPackageContent = apps.get_model('python', 'PythonPackageContent') 9 | RemoteArtifact = apps.get_model('core', 'RemoteArtifact') 10 | package_bulk = [] 11 | for python_package in PythonPackageContent.objects.only("pk", "sha256").iterator(): 12 | content_artifact = python_package.contentartifact_set.first() 13 | if content_artifact.artifact: 14 | artifact = content_artifact.artifact 15 | else: 16 | artifact = RemoteArtifact.objects.filter(content_artifact=content_artifact).first() 17 | python_package.sha256 = artifact.sha256 18 | package_bulk.append(python_package) 19 | if len(package_bulk) == 100000: 20 | with transaction.atomic(): 21 | PythonPackageContent.objects.bulk_update(package_bulk, ["sha256",]) 22 | package_bulk = [] 23 | with transaction.atomic(): 24 | PythonPackageContent.objects.bulk_update(package_bulk, ["sha256",]) 25 | 26 | 27 | class Migration(migrations.Migration): 28 | 29 | dependencies = [ 30 | ('python', '0004_DATA_swap_distribution_model'), 31 | ] 32 | 33 | operations = [ 34 | migrations.AddField( 35 | model_name='pythonpackagecontent', 36 | name='sha256', 37 | field=models.CharField(max_length=64, default=''), 38 | preserve_default=False, 39 | ), 40 | migrations.RunPython(add_sha256_to_current_models, migrations.RunPython.noop) 41 | ] 42 | -------------------------------------------------------------------------------- /.ci/scripts/cherrypick.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # WARNING: DO NOT EDIT! 4 | # 5 | # This file was generated by plugin_template, and is managed by it. Please use 6 | # './plugin-template --github pulp_python' to update this file. 7 | # 8 | # For more info visit https://github.com/pulp/plugin_template 9 | 10 | set -e 11 | 12 | if [ $# -lt 3 ] 13 | then 14 | echo "Usage: .ci/scripts/cherrypick.sh [commit-hash] [original-issue-id] [backport-issue-id]" 15 | echo " ex: .ci/scripts/cherrypick.sh abcd1234 1234 4567" 16 | echo "" 17 | echo "Note: make sure you are on a fork of the release branch before running this script." 18 | exit 19 | fi 20 | 21 | commit="$(git rev-parse $1)" 22 | issue="$2" 23 | backport="$3" 24 | commit_message=$(git log --format=%B -n 1 $commit) 25 | 26 | if ! echo $commit_message | tr '[:upper:]' '[:lower:]' | grep -q "\[noissue\]" 27 | then 28 | if ! echo $commit_message | tr '[:upper:]' '[:lower:]' | grep -q -E "(fixes|closes).*#$issue" 29 | then 30 | echo "Error: issue $issue not detected in commit message." && exit 1 31 | fi 32 | fi 33 | 34 | if [ "$4" = "--continue" ] 35 | then 36 | echo "Continue after manually resolving conflicts..." 37 | elif [ "$4" = "" ] 38 | then 39 | if ! git cherry-pick --no-commit "$commit" 40 | then 41 | echo "Please resolve and add merge conflicts and restart this command with appended '--continue'." 42 | exit 1 43 | fi 44 | else 45 | exit 1 46 | fi 47 | 48 | for file in $(find CHANGES -name "$issue.*") 49 | do 50 | newfile="${file/$issue/$backport}" 51 | git mv "$file" "$newfile" 52 | sed -i -e "\$a (backported from #$issue)" "$newfile" 53 | git add "$newfile" 54 | done 55 | 56 | commit_message="$(printf "$commit_message" | sed -E 's/(fixes|closes)/backports/i')" 57 | commit_message="$commit_message 58 | 59 | fixes #$backport 60 | 61 | (cherry picked from commit $commit)" 62 | git commit -m "$commit_message" 63 | 64 | printf "\nSuccessfully backported commit $1.\n" 65 | -------------------------------------------------------------------------------- /.github/workflows/scripts/install.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # WARNING: DO NOT EDIT! 4 | # 5 | # This file was generated by plugin_template, and is managed by it. Please use 6 | # './plugin-template --github pulp_python' to update this file. 7 | # 8 | # For more info visit https://github.com/pulp/plugin_template 9 | 10 | # make sure this script runs at the repo root 11 | cd "$(dirname "$(realpath -e "$0")")"/../../.. 12 | REPO_ROOT="$PWD" 13 | 14 | set -euv 15 | 16 | source .github/workflows/scripts/utils.sh 17 | 18 | if [[ "$TEST" = "docs" || "$TEST" = "publish" ]]; then 19 | pip install -r ../pulpcore/doc_requirements.txt 20 | pip install -r doc_requirements.txt 21 | fi 22 | 23 | pip install -r functest_requirements.txt 24 | 25 | cd .ci/ansible/ 26 | 27 | TAG=ci_build 28 | if [[ "$TEST" == "plugin-from-pypi" ]]; then 29 | PLUGIN_NAME=pulp_python 30 | elif [[ "${RELEASE_WORKFLOW:-false}" == "true" ]]; then 31 | PLUGIN_NAME=./pulp_python/dist/pulp_python-$PLUGIN_VERSION-py3-none-any.whl 32 | else 33 | PLUGIN_NAME=./pulp_python 34 | fi 35 | if [[ "${RELEASE_WORKFLOW:-false}" == "true" ]]; then 36 | # Install the plugin only and use published PyPI packages for the rest 37 | # Quoting ${TAG} ensures Ansible casts the tag as a string. 38 | cat >> vars/main.yaml << VARSYAML 39 | image: 40 | name: pulp 41 | tag: "${TAG}" 42 | plugins: 43 | - name: pulpcore 44 | source: pulpcore 45 | - name: pulp_python 46 | source: "${PLUGIN_NAME}" 47 | services: 48 | - name: pulp 49 | image: "pulp:${TAG}" 50 | volumes: 51 | - ./settings:/etc/pulp 52 | VARSYAML 53 | else 54 | cat >> vars/main.yaml << VARSYAML 55 | image: 56 | name: pulp 57 | tag: "${TAG}" 58 | plugins: 59 | - name: pulp_python 60 | source: "${PLUGIN_NAME}" 61 | - name: pulpcore 62 | source: ./pulpcore 63 | services: 64 | - name: pulp 65 | image: "pulp:${TAG}" 66 | volumes: 67 | - ./settings:/etc/pulp 68 | VARSYAML 69 | fi 70 | 71 | cat >> vars/main.yaml << VARSYAML 72 | pulp_settings: null 73 | VARSYAML 74 | 75 | ansible-playbook build_container.yaml 76 | ansible-playbook start_container.yaml 77 | 78 | echo ::group::PIP_LIST 79 | cmd_prefix bash -c "pip3 list && pip3 install pipdeptree && pipdeptree" 80 | echo ::endgroup:: 81 | -------------------------------------------------------------------------------- /COMMITMENT: -------------------------------------------------------------------------------- 1 | GPL Cooperation Commitment, version 1.0 2 | 3 | Before filing or continuing to prosecute any legal proceeding or claim 4 | (other than a Defensive Action) arising from termination of a Covered 5 | License, we commit to extend to the person or entity ('you') accused 6 | of violating the Covered License the following provisions regarding 7 | cure and reinstatement, taken from GPL version 3. As used here, the 8 | term 'this License' refers to the specific Covered License being 9 | enforced. 10 | 11 | However, if you cease all violation of this License, then your 12 | license from a particular copyright holder is reinstated (a) 13 | provisionally, unless and until the copyright holder explicitly 14 | and finally terminates your license, and (b) permanently, if the 15 | copyright holder fails to notify you of the violation by some 16 | reasonable means prior to 60 days after the cessation. 17 | 18 | Moreover, your license from a particular copyright holder is 19 | reinstated permanently if the copyright holder notifies you of the 20 | violation by some reasonable means, this is the first time you 21 | have received notice of violation of this License (for any work) 22 | from that copyright holder, and you cure the violation prior to 30 23 | days after your receipt of the notice. 24 | 25 | We intend this Commitment to be irrevocable, and binding and 26 | enforceable against us and assignees of or successors to our 27 | copyrights. 28 | 29 | Definitions 30 | 31 | 'Covered License' means the GNU General Public License, version 2 32 | (GPLv2), the GNU Lesser General Public License, version 2.1 33 | (LGPLv2.1), or the GNU Library General Public License, version 2 34 | (LGPLv2), all as published by the Free Software Foundation. 35 | 36 | 'Defensive Action' means a legal proceeding or claim that We bring 37 | against you in response to a prior proceeding or claim initiated by 38 | you or your affiliate. 39 | 40 | 'We' means each contributor to this repository as of the date of 41 | inclusion of this file, including subsidiaries of a corporate 42 | contributor. 43 | 44 | This work is available under a [Creative Commons Attribution-ShareAlike 4.0 International license] 45 | (https://creativecommons.org/licenses/by-sa/4.0/). 46 | -------------------------------------------------------------------------------- /CONTRIBUTING.rst: -------------------------------------------------------------------------------- 1 | Community 2 | --------- 3 | 4 | This plugin exists to serve the community. If we can do more for your use case, please let us know! 5 | Also, contributions are greatly appreciated in the form of: 6 | 7 | 1. `Github Issues `_ 8 | 2. `Github Pull Requests `_ 9 | 3. `Helping other users `_ 10 | 11 | We can usually be found on Matrix in `#pulp-dev`, `#pulp`, and `#pulp-python`. 12 | 13 | Contributing 14 | ============ 15 | 16 | To contribute to the ``pulp_python`` package follow this process: 17 | 18 | 1. Clone the GitHub repo 19 | 2. Make a change 20 | 3. Make sure all tests passed 21 | 4. Add a file into CHANGES folder (Changelog update). 22 | 5. Commit changes to own ``pulp_python`` clone 23 | 6. Make pull request from github page for your clone against master branch 24 | 25 | 26 | .. _changelog-update: 27 | 28 | Changelog update 29 | **************** 30 | 31 | The CHANGES.rst file is managed using the `towncrier tool `_ 32 | and all non trivial changes must be accompanied by a news entry. 33 | 34 | To add an entry to the news file, you first need an issue in pulp.plan.io describing the change you 35 | want to make. Once you have an issue, take its number and create a file inside of the ``CHANGES/`` 36 | directory named after that issue number with an extension of .feature, .bugfix, .doc, .removal, or 37 | .misc. So if your issue is 3543 and it fixes a bug, you would create the file 38 | ``CHANGES/3543.bugfix``. 39 | 40 | PRs can span multiple categories by creating multiple files (for instance, if you added a feature 41 | and deprecated an old feature at the same time, you would create CHANGES/NNNN.feature and 42 | CHANGES/NNNN.removal). Likewise if a PR touches multiple issues/PRs you may create a file for each 43 | of them with the exact same contents and Towncrier will deduplicate them. 44 | 45 | The contents of this file are reStructuredText formatted text that will be used as the content of 46 | the news file entry. You do not need to reference the issue or PR numbers here as towncrier will 47 | automatically add a reference to all of the affected issues when rendering the news file. 48 | -------------------------------------------------------------------------------- /.ci/scripts/validate_commit_message.py: -------------------------------------------------------------------------------- 1 | # WARNING: DO NOT EDIT! 2 | # 3 | # This file was generated by plugin_template, and is managed by it. Please use 4 | # './plugin-template --github pulp_python' to update this file. 5 | # 6 | # For more info visit https://github.com/pulp/plugin_template 7 | 8 | import re 9 | import subprocess 10 | import sys 11 | from pathlib import Path 12 | 13 | 14 | from github import Github 15 | 16 | 17 | NO_ISSUE = "[noissue]" 18 | CHANGELOG_EXTS = [".feature", ".bugfix", ".doc", ".removal", ".misc", ".deprecation"] 19 | 20 | 21 | KEYWORDS = ["fixes", "closes"] 22 | 23 | sha = sys.argv[1] 24 | message = subprocess.check_output(["git", "log", "--format=%B", "-n 1", sha]).decode("utf-8") 25 | g = Github() 26 | repo = g.get_repo("pulp/pulp_python") 27 | 28 | 29 | def __check_status(issue): 30 | gi = repo.get_issue(int(issue)) 31 | if gi.pull_request: 32 | sys.exit(f"Error: issue #{issue} is a pull request.") 33 | if gi.closed_at: 34 | sys.exit(f"Error: issue #{issue} is closed.") 35 | 36 | 37 | def __check_changelog(issue): 38 | matches = list(Path("CHANGES").rglob(f"{issue}.*")) 39 | 40 | if len(matches) < 1: 41 | sys.exit(f"Could not find changelog entry in CHANGES/ for {issue}.") 42 | for match in matches: 43 | if match.suffix not in CHANGELOG_EXTS: 44 | sys.exit(f"Invalid extension for changelog entry '{match}'.") 45 | 46 | 47 | print("Checking commit message for {sha}.".format(sha=sha[0:7])) 48 | 49 | # validate the issue attached to the commit 50 | regex = r"(?:{keywords})[\s:]+#(\d+)".format(keywords=("|").join(KEYWORDS)) 51 | pattern = re.compile(regex, re.IGNORECASE) 52 | 53 | issues = pattern.findall(message) 54 | 55 | if issues: 56 | for issue in pattern.findall(message): 57 | __check_status(issue) 58 | __check_changelog(issue) 59 | else: 60 | if NO_ISSUE in message: 61 | print("Commit {sha} has no issues but is tagged {tag}.".format(sha=sha[0:7], tag=NO_ISSUE)) 62 | else: 63 | sys.exit( 64 | "Error: no attached issues found for {sha}. If this was intentional, add " 65 | " '{tag}' to the commit message.".format(sha=sha[0:7], tag=NO_ISSUE) 66 | ) 67 | 68 | print("Commit message for {sha} passed.".format(sha=sha[0:7])) 69 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | pulp-python Plugin 2 | ================== 3 | 4 | The `python`` plugin extends `pulpcore `__ to support 5 | hosting python packages. This plugin is a part of the `Pulp Project 6 | `_, and assumes some familiarity with the `pulpcore documentation 7 | `_. 8 | 9 | If you are just getting started, we recommend getting to know the :doc:`basic 10 | workflows`. 11 | 12 | The REST API documentation for ``pulp_python`` is available `here `_. 13 | 14 | Features 15 | -------- 16 | 17 | * :ref:`Create local mirrors of PyPI ` that you have full control over 18 | * :ref:`Upload your own Python packages ` 19 | * :ref:`Perform pip install ` from your Pulp Python repositories 20 | * :ref:`Download packages on-demand ` to reduce disk usage 21 | * Every operation creates a restorable snapshot with :ref:`Versioned Repositories ` 22 | * Curate your Python repositories with allowed and disallowed packages 23 | * Host content either `locally or on S3 `_ 24 | * De-duplication of all saved content 25 | 26 | Tech Preview 27 | ------------ 28 | 29 | Some additional features are being supplied as a tech preview. There is a possibility that 30 | backwards incompatible changes will be introduced for these particular features. For a list of 31 | features currently being supplied as tech previews only, see the :doc:`tech preview page 32 | `. 33 | 34 | How to use these docs 35 | ===================== 36 | 37 | The documentation here should be considered the **primary documentation for managing Python related content.** 38 | All relevent workflows are covered here, with references to some pulpcore supplemental docs. 39 | Users may also find pulpcore’s conceptual docs useful. 40 | 41 | This documentation falls into two main categories: 42 | 43 | 1. :ref:`workflows-index` shows the **major features** of the Python plugin, with links to reference docs. 44 | 2. The `REST API Docs `_ are automatically generated and provide more detailed information for each 45 | minor feature, including all fields and options. 46 | 47 | Table of Contents 48 | ----------------- 49 | 50 | .. toctree:: 51 | :maxdepth: 1 52 | 53 | installation 54 | workflows/index 55 | changes 56 | contributing 57 | restapi/index 58 | tech-preview 59 | 60 | 61 | Indices and tables 62 | ================== 63 | 64 | * :ref:`genindex` 65 | * :ref:`modindex` 66 | * :ref:`search` 67 | 68 | -------------------------------------------------------------------------------- /docs/workflows/pypi.rst: -------------------------------------------------------------------------------- 1 | .. _pypi-workflow: 2 | 3 | Setup your own PyPI: 4 | ==================== 5 | 6 | This section guides you through the quickest way to to setup ``pulp_python`` to act as your very own 7 | private ``PyPI``. 8 | 9 | Create a Repository: 10 | -------------------- 11 | 12 | Repositories are the base objects ``Pulp`` uses to store and organize its content. They are automatically 13 | versioned when content is added or deleted and allow for easy rollbacks to previous versions. 14 | 15 | .. literalinclude:: ../_scripts/repo.sh 16 | :language: bash 17 | 18 | Repository Create Response:: 19 | 20 | { 21 | "pulp_href": "/pulp/api/v3/repositories/python/python/3fe0d204-217f-4250-8177-c83b30751fca/", 22 | "pulp_created": "2021-06-02T14:54:53.387054Z", 23 | "versions_href": "/pulp/api/v3/repositories/python/python/3fe0d204-217f-4250-8177-c83b30751fca/versions/", 24 | "pulp_labels": {}, 25 | "latest_version_href": "/pulp/api/v3/repositories/python/python/3fe0d204-217f-4250-8177-c83b30751fca/versions/1/", 26 | "name": "foo", 27 | "description": null, 28 | "retained_versions": null, 29 | "remote": null, 30 | "autopublish": false 31 | } 32 | 33 | Create a Distribution: 34 | ---------------------- 35 | 36 | Distributions serve the content stored in repositories so that it can be used by tools like ``pip``. 37 | 38 | .. literalinclude:: ../_scripts/index.sh 39 | :language: bash 40 | 41 | Distribution Create Response:: 42 | 43 | { 44 | "pulp_href": "/pulp/api/v3/distributions/python/pypi/e8438593-fd40-4654-8577-65398833c331/", 45 | "pulp_created": "2021-06-03T20:04:18.233230Z", 46 | "base_path": "my-pypi", 47 | "base_url": "https://pulp3-source-fedora33.localhost.example.com/pypi/foo/", 48 | "content_guard": null, 49 | "pulp_labels": {}, 50 | "name": "my-pypi", 51 | "repository": "/pulp/api/v3/repositories/python/python/3fe0d204-217f-4250-8177-c83b30751fca/", 52 | "publication": null, 53 | "allow_uploads": true 54 | } 55 | 56 | Upload and Install Packages: 57 | ---------------------------- 58 | 59 | Packages can now be uploaded to the index using your favorite Python tool. The index url will be available 60 | at ``/pypi//simple/``. 61 | 62 | .. literalinclude:: ../_scripts/twine.sh 63 | :language: bash 64 | 65 | Packages can then be installed using your favorite Python tool:: 66 | 67 | $ pip install --trusted-host localhost -i $BASE_ADDR/pypi/my-pypi/simple/ shelf-reader 68 | 69 | Now you have a fully operational Python package index. Check out the other workflows to see more features of 70 | ``pulp_python``. 71 | -------------------------------------------------------------------------------- /docs/installation.rst: -------------------------------------------------------------------------------- 1 | User Setup 2 | ========== 3 | 4 | All workflow examples use the Pulp CLI. Install and setup from PyPI: 5 | 6 | .. code-block:: bash 7 | 8 | pip install pulp-cli[pygments] # For color output 9 | pulp config create -e 10 | pulp status # Check that CLI can talk to Pulp 11 | 12 | If you configured the ``admin`` user with a different password, adjust the configuration 13 | accordingly. If you prefer to specify the username and password with each request, please see 14 | ``Pulp CLI`` documentation on how to do that. 15 | 16 | 17 | Install ``pulpcore`` 18 | -------------------- 19 | 20 | Follow the `installation 21 | instructions `__ 22 | provided with pulpcore. 23 | 24 | Install plugin 25 | -------------- 26 | 27 | This document assumes that you have 28 | `installed pulpcore `_ 29 | into a the virtual environment ``pulpvenv``. 30 | 31 | Users should install from **either** PyPI or source. 32 | 33 | From Source 34 | *********** 35 | 36 | .. code-block:: bash 37 | 38 | sudo -u pulp -i 39 | source ~/pulpvenv/bin/activate 40 | cd pulp_python 41 | pip install -e . 42 | django-admin runserver 24817 43 | 44 | Make and Run Migrations 45 | ----------------------- 46 | 47 | .. code-block:: bash 48 | 49 | pulpcore-manager makemigrations python 50 | pulpcore-manager migrate python 51 | 52 | (Optional) Configure the google pub/sub service 53 | ----------------------------------------------- 54 | 55 | Create a service account in the google cloud console and make sure to give it the pubsub role. 56 | Once created, download the JSON key file and save it somewhere on disk. Type 57 | ``sudo systemctl edit pulpcore-content`` in your shell to create a configuration file for the 58 | pulp-content service and copy this content into it. 59 | 60 | .. code-block:: bash 61 | 62 | [Service] 63 | Environment="GOOGLE_APPLICATION_CREDENTIALS=" 64 | 65 | Go back to the google cloud console and create a new topic for your application. Open the django 66 | configuration file (Should live under ``/etc/pulp/settings.py``) for pulp and add the values for 67 | ``GOOGLE_PUBSUB_PROJECT_ID`` and ``GOOGLE_PUBSUB_TOPIC_ID``, which should be the label for the project 68 | in which you created your pub/sub topic and the topic label itself respectively. 69 | 70 | Run Services 71 | ------------ 72 | 73 | .. code-block:: bash 74 | 75 | pulpcore-manager runserver 76 | gunicorn pulpcore.content:server --bind 'localhost:24816' --worker-class 'aiohttp.GunicornWebWorker' -w 2 77 | sudo systemctl restart pulpcore-resource-manager 78 | sudo systemctl restart pulpcore-worker@1 79 | sudo systemctl restart pulpcore-worker@2 80 | -------------------------------------------------------------------------------- /pulp_python/tests/functional/api/test_full_mirror.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | """Tests that python plugin can fully mirror PyPi and other Pulp repositories""" 3 | import unittest 4 | 5 | from pulp_smash import config 6 | from pulp_smash.pulp3.bindings import monitor_task 7 | from pulp_smash.pulp3.utils import gen_repo, get_content_summary 8 | 9 | from pulp_python.tests.functional.constants import PYTHON_CONTENT_NAME 10 | from pulp_python.tests.functional.utils import gen_python_client, gen_python_remote 11 | from pulp_python.tests.functional.utils import set_up_module as setUpModule # noqa:F401 12 | 13 | from pulpcore.client.pulpcore import TasksApi, ApiClient as CoreApiClient, Configuration 14 | from pulpcore.client.pulp_python import ( 15 | RepositoriesPythonApi, 16 | RepositorySyncURL, 17 | RemotesPythonApi, 18 | ) 19 | import socket 20 | 21 | 22 | @unittest.skip 23 | class PyPiMirrorTestCase(unittest.TestCase): 24 | """ 25 | Testing Pulp's full syncing ability of Python repositories 26 | 27 | This test targets the following issues: 28 | 29 | * `Pulp #985 `_ 30 | """ 31 | 32 | @classmethod 33 | def setUpClass(cls): 34 | """Create class-wide variables.""" 35 | cls.cfg = config.get_config() 36 | cls.client = gen_python_client() 37 | configuration = Configuration() 38 | configuration.username = 'admin' 39 | configuration.password = 'password' 40 | configuration.host = 'http://{}:24817'.format(socket.gethostname()) 41 | configuration.safe_chars_for_path_param = '/' 42 | cls.core_client = CoreApiClient(configuration) 43 | 44 | def test_on_demand_pypi_full_sync(self): 45 | """This test syncs all of PyPi""" 46 | repo_api = RepositoriesPythonApi(self.client) 47 | remote_api = RemotesPythonApi(self.client) 48 | tasks_api = TasksApi(self.core_client) 49 | 50 | repo = repo_api.create(gen_repo()) 51 | self.addCleanup(repo_api.delete, repo.pulp_href) 52 | 53 | body = gen_python_remote("https://pypi.org", includes=[], policy="on_demand") 54 | remote = remote_api.create(body) 55 | self.addCleanup(remote_api.delete, remote.pulp_href) 56 | 57 | # Sync the repository. 58 | self.assertEqual(repo.latest_version_href, f"{repo.pulp_href}versions/0/") 59 | repository_sync_data = RepositorySyncURL(remote=remote.pulp_href) 60 | sync_response = repo_api.sync(repo.pulp_href, repository_sync_data) 61 | monitor_task(sync_response.task) 62 | repo = repo_api.read(repo.pulp_href) 63 | 64 | sync_task = tasks_api.read(sync_response.task) 65 | time_diff = sync_task.finished_at - sync_task.started_at 66 | print("Delete time: {} seconds".format(time_diff.seconds)) 67 | 68 | self.assertIsNotNone(repo.latest_version_href) 69 | # As of August 11 2020, all_packages() returns 253,587 packages, 70 | # only 248,677 of them were downloadable 71 | self.assertTrue(get_content_summary(repo.to_dict())[PYTHON_CONTENT_NAME] > 245000) 72 | -------------------------------------------------------------------------------- /pulp_python/tests/functional/api/test_auto_publish.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | """Tests automatic updating of publications and distributions.""" 3 | from pulp_smash.pulp3.bindings import monitor_task 4 | from pulp_smash.pulp3.utils import download_content_unit 5 | 6 | from pulp_python.tests.functional.utils import ( 7 | cfg, 8 | gen_python_remote, 9 | TestCaseUsingBindings, 10 | TestHelpersMixin 11 | ) 12 | from pulp_python.tests.functional.utils import set_up_module as setUpModule # noqa:F401 13 | from pulpcore.client.pulp_python import RepositorySyncURL 14 | 15 | 16 | class AutoPublishDistributeTestCase(TestCaseUsingBindings, TestHelpersMixin): 17 | """Test auto-publish and auto-distribution""" 18 | 19 | def setUp(self): 20 | """Create remote, repo, publish settings, and distribution.""" 21 | self.remote = self.remote_api.create(gen_python_remote(policy="immediate")) 22 | self.addCleanup(self.remote_api.delete, self.remote.pulp_href) 23 | self.repo, self.distribution = self._create_empty_repo_and_distribution(autopublish=True) 24 | 25 | def test_01_sync(self): 26 | """Assert that syncing the repository triggers auto-publish and auto-distribution.""" 27 | self.assertEqual(self.publications_api.list().count, 0) 28 | self.assertTrue(self.distribution.publication is None) 29 | 30 | # Sync the repository. 31 | repository_sync_data = RepositorySyncURL(remote=self.remote.pulp_href) 32 | sync_response = self.repo_api.sync(self.repo.pulp_href, repository_sync_data) 33 | task = monitor_task(sync_response.task) 34 | 35 | # Check that all the appropriate resources were created 36 | self.assertGreater(len(task.created_resources), 1) 37 | self.assertEqual(self.publications_api.list().count, 1) 38 | download_content_unit(cfg, self.distribution.to_dict(), "simple/") 39 | 40 | # Sync the repository again. Since there should be no new repository version, there 41 | # should be no new publications or distributions either. 42 | sync_response = self.repo_api.sync(self.repo.pulp_href, repository_sync_data) 43 | task = monitor_task(sync_response.task) 44 | 45 | self.assertEqual(len(task.created_resources), 0) 46 | self.assertEqual(self.publications_api.list().count, 1) 47 | 48 | def test_02_modify(self): 49 | """Assert that modifying the repository triggers auto-publish and auto-distribution.""" 50 | self.assertEqual(self.publications_api.list().count, 0) 51 | self.assertTrue(self.distribution.publication is None) 52 | 53 | # Modify the repository by adding a content unit 54 | content = self.content_api.list().results[0].pulp_href 55 | modify_response = self.repo_api.modify( 56 | self.repo.pulp_href, {"add_content_units": [content]} 57 | ) 58 | task = monitor_task(modify_response.task) 59 | 60 | # Check that all the appropriate resources were created 61 | self.assertGreater(len(task.created_resources), 1) 62 | self.assertEqual(self.publications_api.list().count, 1) 63 | download_content_unit(cfg, self.distribution.to_dict(), "simple/") 64 | -------------------------------------------------------------------------------- /docs/workflows/publish.rst: -------------------------------------------------------------------------------- 1 | .. _host: 2 | 3 | Publish and Host 4 | ================ 5 | 6 | This section assumes that you have a repository with content in it. To do this, see the 7 | :doc:`sync` or :doc:`upload` documentation. 8 | 9 | Create a Publication (manually) 10 | ------------------------------- 11 | 12 | Kick off a publish task by creating a new publication. The publish task will generate all the 13 | metadata that ``pip`` needs to install packages (although it will need to be hosted through a 14 | Distribution before it is consumable). 15 | 16 | .. literalinclude:: ../_scripts/publication.sh 17 | :language: bash 18 | 19 | Publication Create Response:: 20 | 21 | { 22 | "pulp_href": "/pulp/api/v3/publications/python/pypi/cad6007d-7172-41d1-8c22-0ec95e1d242a/", 23 | "pulp_created": "2021-03-09T04:30:16.686784Z", 24 | "repository_version": "/pulp/api/v3/repositories/python/python/8fbb24ee-dc91-44f4-a6ee-beec60aa542d/versions/1/", 25 | "repository": "/pulp/api/v3/repositories/python/python/8fbb24ee-dc91-44f4-a6ee-beec60aa542d/", 26 | "distributions": [] 27 | } 28 | 29 | Host a Publication (Create a Distribution) 30 | -------------------------------------------- 31 | 32 | To host a publication, (which makes it consumable by ``pip``), users create a distribution which 33 | will serve the associated publication at ``/pypi//`` 34 | 35 | .. literalinclude:: ../_scripts/distribution.sh 36 | :language: bash 37 | 38 | Response:: 39 | 40 | { 41 | "pulp_href": "/pulp/api/v3/distributions/python/pypi/4839c056-6f2b-46b9-ac5f-88eb8a7739a5/", 42 | "pulp_created": "2021-03-09T04:36:48.289737Z", 43 | "base_path": "foo", 44 | "base_url": "/pypi/foo/", 45 | "content_guard": null, 46 | "pulp_labels": {}, 47 | "name": "foo", 48 | "publication": "/pulp/api/v3/publications/python/pypi/a09111b1-6bce-43ac-aed7-2e8441c22704/" 49 | } 50 | 51 | Automate Publication and Distribution 52 | ------------------------------------- 53 | 54 | With a little more initial setup, you can have publications and distributions for your repositories 55 | updated automatically when new repository versions are created. 56 | 57 | .. literalinclude:: ../_scripts/autoupdate.sh 58 | :language: bash 59 | 60 | .. warning:: 61 | Support for automatic publication and distribution is provided as a tech preview in Pulp 3. 62 | Functionality may not work or may be incomplete. Also, backwards compatibility when upgrading 63 | is not guaranteed. 64 | 65 | .. _using-distributions: 66 | 67 | Use the newly created distribution 68 | ----------------------------------- 69 | 70 | The metadata and packages can now be retrieved from the distribution:: 71 | 72 | $ http $BASE_ADDR/pypi/foo/simple/ 73 | $ http $BASE_ADDR/pypi/foo/simple/shelf-reader/ 74 | 75 | The content is also pip installable:: 76 | 77 | $ pip install --trusted-host localhost -i $BASE_ADDR/pypi/foo/simple/ shelf-reader 78 | 79 | If you don't want to specify the distribution path every time, you can modify your ``pip.conf`` 80 | file. See the `pip docs `_ for more 81 | detail.:: 82 | 83 | $ cat pip.conf 84 | 85 | .. code:: 86 | 87 | [global] 88 | index-url = http://localhost:24817/pypi/foo/simple/ 89 | 90 | The above configuration informs ``pip`` to install from ``pulp``:: 91 | 92 | $ pip install --trusted-host localhost shelf-reader 93 | -------------------------------------------------------------------------------- /pulp_python/app/pypi/serializers.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from gettext import gettext as _ 3 | 4 | from rest_framework import serializers 5 | from pulp_python.app.tasks.upload import DIST_EXTENSIONS 6 | from pulpcore.plugin.models import Artifact 7 | from django.db.utils import IntegrityError 8 | 9 | log = logging.getLogger(__name__) 10 | 11 | 12 | class SummarySerializer(serializers.Serializer): 13 | """ 14 | A Serializer for summary information of an index. 15 | """ 16 | 17 | projects = serializers.IntegerField(help_text=_("Number of Python projects in index")) 18 | releases = serializers.IntegerField( 19 | help_text=_("Number of Python distribution releases in index") 20 | ) 21 | files = serializers.IntegerField(help_text=_("Number of files for all distributions in index")) 22 | 23 | 24 | class PackageMetadataSerializer(serializers.Serializer): 25 | """ 26 | A Serializer for a package's metadata. 27 | """ 28 | 29 | last_serial = serializers.IntegerField(help_text=_("Cache value from last PyPI sync")) 30 | info = serializers.JSONField(help_text=_("Core metadata of the package")) 31 | releases = serializers.JSONField(help_text=_("List of all the releases of the package")) 32 | urls = serializers.JSONField() 33 | 34 | 35 | class PackageUploadSerializer(serializers.Serializer): 36 | """ 37 | A Serializer for Python packages being uploaded to the index. 38 | """ 39 | 40 | content = serializers.FileField( 41 | help_text=_("A Python package release file to upload to the index."), 42 | required=True, 43 | write_only=True, 44 | ) 45 | action = serializers.CharField( 46 | help_text=_("Defaults to `file_upload`, don't change it or request will fail!"), 47 | default="file_upload", 48 | source=":action" 49 | ) 50 | sha256_digest = serializers.CharField( 51 | help_text=_("SHA256 of package to validate upload integrity."), 52 | required=True, 53 | min_length=64, 54 | max_length=64, 55 | ) 56 | 57 | def validate(self, data): 58 | """Validates the request.""" 59 | action = data.get(":action") 60 | if action != "file_upload": 61 | raise serializers.ValidationError( 62 | _("We do not support the :action {}").format(action) 63 | ) 64 | file = data.get("content") 65 | for ext, packagetype in DIST_EXTENSIONS.items(): 66 | if file.name.endswith(ext): 67 | break 68 | else: 69 | raise serializers.ValidationError(_( 70 | "Extension on {} is not a valid python extension " 71 | "(.whl, .exe, .egg, .tar.gz, .tar.bz2, .zip)").format(file.name) 72 | ) 73 | sha256 = data.get("sha256_digest") 74 | digests = {"sha256": sha256} if sha256 else None 75 | artifact = Artifact.init_and_validate(file, expected_digests=digests) 76 | try: 77 | artifact.save() 78 | except IntegrityError: 79 | artifact = Artifact.objects.get(sha256=artifact.sha256) 80 | log.info(f"Artifact for {file.name} already existed in database") 81 | data["content"] = (artifact, file.name) 82 | return data 83 | 84 | 85 | class PackageUploadTaskSerializer(serializers.Serializer): 86 | """ 87 | A Serializer for responding to a package upload task. 88 | """ 89 | 90 | session = serializers.CharField() 91 | task = serializers.CharField() 92 | task_start_time = serializers.DateTimeField() 93 | -------------------------------------------------------------------------------- /pulp_python/app/migrations/0004_DATA_swap_distribution_model.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.19 on 2021-03-19 17:42 2 | 3 | from django.db import migrations, models, transaction 4 | import django.db.models.deletion 5 | 6 | 7 | def migrate_data_from_old_model_to_new_model_up(apps, schema_editor): 8 | """ Move objects from PythonDistribution to NewPythonDistribution.""" 9 | PythonDistribution = apps.get_model('python', 'PythonDistribution') 10 | NewPythonDistribution = apps.get_model('python', 'NewPythonDistribution') 11 | for python_distribution in PythonDistribution.objects.all(): 12 | with transaction.atomic(): 13 | NewPythonDistribution( 14 | pulp_id=python_distribution.pulp_id, 15 | pulp_created=python_distribution.pulp_created, 16 | pulp_last_updated=python_distribution.pulp_last_updated, 17 | pulp_type=python_distribution.pulp_type, 18 | name=python_distribution.name, 19 | base_path=python_distribution.base_path, 20 | content_guard=python_distribution.content_guard, 21 | remote=python_distribution.remote, 22 | publication=python_distribution.publication 23 | ).save() 24 | python_distribution.delete() 25 | 26 | 27 | def migrate_data_from_old_model_to_new_model_down(apps, schema_editor): 28 | """ Move objects from NewPythonDistribution to PythonDistribution.""" 29 | PythonDistribution = apps.get_model('python', 'PythonDistribution') 30 | NewPythonDistribution = apps.get_model('python', 'NewPythonDistribution') 31 | for python_distribution in NewPythonDistribution.objects.all(): 32 | with transaction.atomic(): 33 | PythonDistribution( 34 | pulp_id=python_distribution.pulp_id, 35 | pulp_created=python_distribution.pulp_created, 36 | pulp_last_updated=python_distribution.pulp_last_updated, 37 | pulp_type=python_distribution.pulp_type, 38 | name=python_distribution.name, 39 | base_path=python_distribution.base_path, 40 | content_guard=python_distribution.content_guard, 41 | remote=python_distribution.remote, 42 | publication=python_distribution.publication 43 | ).save() 44 | python_distribution.delete() 45 | 46 | 47 | class Migration(migrations.Migration): 48 | atomic = False 49 | 50 | dependencies = [ 51 | ('core', '0062_add_new_distribution_mastermodel'), 52 | ('python', '0003_new_sync_filters'), 53 | ] 54 | 55 | operations = [ 56 | migrations.CreateModel( 57 | name='NewPythonDistribution', 58 | fields=[ 59 | ('distribution_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, related_name='python_pythondistribution', serialize=False, to='core.Distribution')), 60 | ], 61 | options={ 62 | 'default_related_name': '%(app_label)s_%(model_name)s', 63 | }, 64 | bases=('core.distribution',), 65 | ), 66 | migrations.RunPython( 67 | code=migrate_data_from_old_model_to_new_model_up, 68 | reverse_code=migrate_data_from_old_model_to_new_model_down, 69 | ), 70 | migrations.DeleteModel( 71 | name='PythonDistribution', 72 | ), 73 | migrations.RenameModel( 74 | old_name='NewPythonDistribution', 75 | new_name='PythonDistribution', 76 | ), 77 | ] 78 | -------------------------------------------------------------------------------- /docs/workflows/upload.rst: -------------------------------------------------------------------------------- 1 | .. _uploading-content: 2 | 3 | Upload and Manage Content 4 | ========================= 5 | Content can be added to a repository not only by synchronizing from a remote source but also by uploading the files directly into Pulp. 6 | 7 | Create a repository 8 | ------------------- 9 | 10 | If you don't already have a repository, create one. 11 | 12 | .. literalinclude:: ../_scripts/repo.sh 13 | :language: bash 14 | 15 | Repository GET response:: 16 | 17 | { 18 | "pulp_href": "/pulp/api/v3/repositories/python/python/8fbb24ee-dc91-44f4-a6ee-beec60aa542d/", 19 | "pulp_created": "2021-03-09T04:11:54.347921Z", 20 | "versions_href": "/pulp/api/v3/repositories/python/python/8fbb24ee-dc91-44f4-a6ee-beec60aa542d/versions/", 21 | "pulp_labels": {}, 22 | "latest_version_href": "/pulp/api/v3/repositories/python/python/8fbb24ee-dc91-44f4-a6ee-beec60aa542d/versions/0/", 23 | "name": "foo", 24 | "description": null, 25 | "remote": null 26 | } 27 | 28 | Upload a file to Pulp 29 | --------------------- 30 | 31 | Each artifact in Pulp represents a file. They can be created during sync or created manually by uploading a file. 32 | 33 | .. literalinclude:: ../_scripts/artifact.sh 34 | :language: bash 35 | 36 | Content Upload response:: 37 | 38 | { 39 | "pulp_href": "/pulp/api/v3/content/python/packages/f226894b-daa9-4152-9a04-595979ea5f9b/", 40 | "pulp_created": "2021-03-09T04:47:13.066911Z", 41 | "artifact": "/pulp/api/v3/artifacts/532b6318-add2-4208-ac1b-d6d37a39a97f/", 42 | "filename": "shelf-reader-0.1.tar.gz", 43 | "packagetype": "sdist", 44 | "name": "shelf-reader", 45 | "version": "0.1", 46 | "metadata_version": "1.1", 47 | "summary": "Make sure your collections are in call number order.", 48 | "description": "too long to read" 49 | "keywords": "", 50 | "home_page": "https://github.com/asmacdo/shelf-reader", 51 | "download_url": "", 52 | "author": "Austin Macdonald", 53 | "author_email": "asmacdo@gmail.com", 54 | "maintainer": "", 55 | "maintainer_email": "", 56 | "license": "GNU GENERAL PUBLIC LICENSE Version 2, June 1991", 57 | "requires_python": "", 58 | "project_url": "", 59 | "platform": "", 60 | "supported_platform": "", 61 | "requires_dist": "[]", 62 | "provides_dist": "[]", 63 | "obsoletes_dist": "[]", 64 | "requires_external": "[]", 65 | "classifiers": "[]" 66 | } 67 | 68 | Add content to a repository 69 | --------------------------- 70 | 71 | Once there is a content unit, it can be added and removed from repositories using the add and remove commands 72 | 73 | .. literalinclude:: ../_scripts/add_content_repo.sh 74 | :language: bash 75 | 76 | Repository Version Show Response (after task complete):: 77 | 78 | { 79 | "base_version": null, 80 | "content_summary": { 81 | "added": { 82 | "python.python": { 83 | "count": 1, 84 | "href": "/pulp/api/v3/content/python/packages/?repository_version_added=/pulp/api/v3/repositories/python/python/931109d3-db86-4933-bf1d-45b4d4216d5d/versions/1/" 85 | } 86 | }, 87 | "present": { 88 | "python.python": { 89 | "count": 1, 90 | "href": "/pulp/api/v3/content/python/packages/?repository_version=/pulp/api/v3/repositories/python/python/931109d3-db86-4933-bf1d-45b4d4216d5d/versions/1/" 91 | } 92 | }, 93 | "removed": {} 94 | }, 95 | "number": 1, 96 | "pulp_created": "2020-05-28T21:04:54.403979Z", 97 | "pulp_href": "/pulp/api/v3/repositories/python/python/931109d3-db86-4933-bf1d-45b4d4216d5d/versions/1/" 98 | } 99 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | # WARNING: DO NOT EDIT! 2 | # 3 | # This file was generated by plugin_template, and is managed by it. Please use 4 | # './plugin-template --github pulp_python' to update this file. 5 | # 6 | # For more info visit https://github.com/pulp/plugin_template 7 | --- 8 | name: Pulp CI 9 | on: 10 | pull_request: 11 | branches: 12 | - '*' 13 | 14 | jobs: 15 | 16 | lint: 17 | runs-on: ubuntu-latest 18 | 19 | steps: 20 | - uses: actions/checkout@v2 21 | with: 22 | # by default, it uses a depth of 1 23 | # this fetches all history so that we can read each commit 24 | fetch-depth: 0 25 | 26 | - uses: actions/setup-python@v2 27 | with: 28 | python-version: "3.7" 29 | 30 | # dev_requirements contains tools needed for flake8, etc. 31 | - name: Install requirements 32 | run: pip3 install -r dev_requirements.txt 33 | 34 | 35 | 36 | 37 | # Lint code. 38 | - name: Run flake8 39 | run: flake8 --config flake8.cfg 40 | 41 | # check for any files unintentionally left out of MANIFEST.in 42 | - name: Check manifest 43 | run: check-manifest 44 | 45 | - name: Check for pulpcore imports outside of pulpcore.plugin 46 | run: sh .ci/scripts/check_pulpcore_imports.sh 47 | 48 | - name: Check for gettext problems 49 | run: sh .ci/scripts/check_gettext.sh 50 | 51 | test: 52 | runs-on: ubuntu-latest 53 | # run only after lint finishes 54 | needs: lint 55 | strategy: 56 | fail-fast: false 57 | matrix: 58 | env: 59 | - TEST: pulp 60 | - TEST: docs 61 | - TEST: s3 62 | 63 | steps: 64 | - uses: actions/checkout@v2 65 | with: 66 | # by default, it uses a depth of 1 67 | # this fetches all history so that we can read each commit 68 | fetch-depth: 0 69 | 70 | - uses: actions/setup-python@v2 71 | with: 72 | python-version: "3.7" 73 | 74 | - name: Install httpie 75 | run: | 76 | echo ::group::HTTPIE 77 | sudo apt-get update -yq 78 | sudo -E apt-get -yq --no-install-suggests --no-install-recommends install httpie 79 | echo ::endgroup:: 80 | echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/.ci/assets/httpie/" >> $GITHUB_ENV 81 | echo "TEST=${{ matrix.env.TEST }}" >> $GITHUB_ENV 82 | 83 | - name: Before Install 84 | run: .github/workflows/scripts/before_install.sh 85 | shell: bash 86 | 87 | - name: Install 88 | run: .github/workflows/scripts/install.sh 89 | env: 90 | PY_COLORS: '1' 91 | ANSIBLE_FORCE_COLOR: '1' 92 | shell: bash 93 | 94 | - name: Install Python client 95 | run: .github/workflows/scripts/install_python_client.sh 96 | 97 | - name: Install Ruby client 98 | if: ${{ env.TEST == 'bindings' }} 99 | run: .github/workflows/scripts/install_ruby_client.sh 100 | 101 | - name: Before Script 102 | run: | 103 | .github/workflows/scripts/before_script.sh 104 | 105 | - name: Setting secrets 106 | if: github.event_name != 'pull_request' 107 | run: python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" 108 | env: 109 | SECRETS_CONTEXT: ${{ toJson(secrets) }} 110 | 111 | - name: Script 112 | run: .github/workflows/scripts/script.sh 113 | shell: bash 114 | 115 | - name: After failure 116 | if: failure() 117 | run: | 118 | echo "Need to debug? Please check: https://github.com/marketplace/actions/debugging-with-tmate" 119 | http --timeout 30 --check-status --pretty format --print hb http://pulp/pulp/api/v3/status/ || true 120 | docker images || true 121 | docker ps -a || true 122 | docker logs pulp || true 123 | docker exec pulp ls -latr /etc/yum.repos.d/ || true 124 | docker exec pulp cat /etc/yum.repos.d/* || true 125 | docker exec pulp pip3 list 126 | -------------------------------------------------------------------------------- /.ci/ansible/start_container.yaml: -------------------------------------------------------------------------------- 1 | # Ansible playbook to start the pulp service container and its supporting services 2 | --- 3 | - hosts: localhost 4 | gather_facts: false 5 | vars_files: 6 | - vars/main.yaml 7 | tasks: 8 | - name: "Create Settings Directories" 9 | file: 10 | path: "{{ item }}" 11 | state: directory 12 | mode: "0755" 13 | loop: 14 | - settings 15 | - ~/.config/pulp_smash 16 | 17 | - name: "Generate Pulp Settings" 18 | template: 19 | src: settings.py.j2 20 | dest: settings/settings.py 21 | 22 | - name: "Configure pulp-smash" 23 | copy: 24 | src: smash-config.json 25 | dest: ~/.config/pulp_smash/settings.json 26 | 27 | - name: "Setup docker networking" 28 | docker_network: 29 | name: pulp_ci_bridge 30 | 31 | - name: "Start Service Containers" 32 | docker_container: 33 | name: "{{ item.name }}" 34 | image: "{{ item.image }}" 35 | auto_remove: true 36 | recreate: true 37 | privileged: false 38 | networks: 39 | - name: pulp_ci_bridge 40 | aliases: "{{ item.name }}" 41 | volumes: "{{ item.volumes | default(omit) }}" 42 | env: "{{ item.env | default(omit) }}" 43 | command: "{{ item.command | default(omit) }}" 44 | state: started 45 | loop: "{{ services | default([]) }}" 46 | 47 | - name: "Retrieve Docker Network Info" 48 | docker_network_info: 49 | name: pulp_ci_bridge 50 | register: pulp_ci_bridge_info 51 | 52 | - name: "Update /etc/hosts" 53 | lineinfile: 54 | path: /etc/hosts 55 | regexp: "\\s{{ item.value.Name }}\\s*$" 56 | line: "{{ item.value.IPv4Address | ipaddr('address') }}\t{{ item.value.Name }}" 57 | loop: "{{ pulp_ci_bridge_info.network.Containers | dict2items }}" 58 | become: true 59 | 60 | - name: "Create Pulp Bucket" 61 | amazon.aws.s3_bucket: 62 | aws_access_key: "{{ minio_access_key }}" 63 | aws_secret_key: "{{ minio_secret_key }}" 64 | s3_url: "http://minio:9000" 65 | region: eu-central-1 66 | name: pulp3 67 | state: present 68 | when: s3_test | default(false) 69 | 70 | - block: 71 | - name: "Wait for Pulp" 72 | uri: 73 | url: "http://pulp/pulp/api/v3/status/" 74 | follow_redirects: none 75 | register: result 76 | until: result.status == 200 77 | retries: 12 78 | delay: 5 79 | rescue: 80 | - name: "Output pulp container log" 81 | command: "docker logs pulp" 82 | failed_when: true 83 | 84 | - block: 85 | - name: "Check version of component being tested" 86 | assert: 87 | that: 88 | - (result.json.versions | items2dict(key_name="component", value_name="version"))[component_name] | canonical_semver == (component_version | canonical_semver) 89 | fail_msg: | 90 | Component {{ component_name }} was expected to be installed in version {{ component_version }}. 91 | Instead it is reported as version {{ (result.json.versions | items2dict(key_name="component", value_name="version"))[component_name] }}. 92 | rescue: 93 | - name: "Check version of component being tested (legacy)" 94 | assert: 95 | that: 96 | - (result.json.versions | items2dict(key_name="component", value_name="version"))[legacy_component_name] | canonical_semver == (component_version | canonical_semver) 97 | fail_msg: | 98 | Component {{ legacy_component_name }} was expected to be installed in version {{ component_version }}. 99 | Instead it is reported as version {{ (result.json.versions | items2dict(key_name="component", value_name="version"))[legacy_component_name] }}. 100 | 101 | - name: "Set pulp password in .netrc" 102 | copy: 103 | dest: "~/.netrc" 104 | content: | 105 | machine pulp 106 | login admin 107 | password password 108 | 109 | - hosts: pulp 110 | gather_facts: false 111 | tasks: 112 | - name: "Set pulp admin password" 113 | command: 114 | cmd: "pulpcore-manager reset-admin-password --password password" 115 | ... 116 | -------------------------------------------------------------------------------- /.github/workflows/scripts/script.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # coding=utf-8 3 | 4 | # WARNING: DO NOT EDIT! 5 | # 6 | # This file was generated by plugin_template, and is managed by it. Please use 7 | # './plugin-template --github pulp_python' to update this file. 8 | # 9 | # For more info visit https://github.com/pulp/plugin_template 10 | 11 | # make sure this script runs at the repo root 12 | cd "$(dirname "$(realpath -e "$0")")"/../../.. 13 | REPO_ROOT="$PWD" 14 | 15 | set -mveuo pipefail 16 | 17 | source .github/workflows/scripts/utils.sh 18 | 19 | export POST_SCRIPT=$PWD/.github/workflows/scripts/post_script.sh 20 | export POST_DOCS_TEST=$PWD/.github/workflows/scripts/post_docs_test.sh 21 | export FUNC_TEST_SCRIPT=$PWD/.github/workflows/scripts/func_test_script.sh 22 | 23 | # Needed for both starting the service and building the docs. 24 | # Gets set in .github/settings.yml, but doesn't seem to inherited by 25 | # this script. 26 | export DJANGO_SETTINGS_MODULE=pulpcore.app.settings 27 | export PULP_SETTINGS=$PWD/.ci/ansible/settings/settings.py 28 | 29 | export PULP_URL="http://pulp" 30 | 31 | if [[ "$TEST" = "docs" ]]; then 32 | cd docs 33 | make PULP_URL="$PULP_URL" diagrams html 34 | tar -cvf docs.tar ./_build 35 | cd .. 36 | 37 | echo "Validating OpenAPI schema..." 38 | cat $PWD/.ci/scripts/schema.py | cmd_stdin_prefix bash -c "cat > /tmp/schema.py" 39 | cmd_prefix bash -c "python3 /tmp/schema.py" 40 | cmd_prefix bash -c "pulpcore-manager spectacular --file pulp_schema.yml --validate" 41 | 42 | if [ -f $POST_DOCS_TEST ]; then 43 | source $POST_DOCS_TEST 44 | fi 45 | exit 46 | fi 47 | 48 | if [[ "${RELEASE_WORKFLOW:-false}" == "true" ]]; then 49 | REPORTED_VERSION=$(http pulp/pulp/api/v3/status/ | jq --arg plugin python --arg legacy_plugin pulp_python -r '.versions[] | select(.component == $plugin or .component == $legacy_plugin) | .version') 50 | response=$(curl --write-out %{http_code} --silent --output /dev/null https://pypi.org/project/pulp-python/$REPORTED_VERSION/) 51 | if [ "$response" == "200" ]; 52 | then 53 | echo "pulp_python $REPORTED_VERSION has already been released. Skipping running tests." 54 | exit 55 | fi 56 | fi 57 | 58 | if [[ "$TEST" == "plugin-from-pypi" ]]; then 59 | COMPONENT_VERSION=$(http https://pypi.org/pypi/pulp-python/json | jq -r '.info.version') 60 | git checkout ${COMPONENT_VERSION} -- pulp_python/tests/ 61 | fi 62 | 63 | cd ../pulp-openapi-generator 64 | ./generate.sh pulpcore python 65 | pip install ./pulpcore-client 66 | rm -rf ./pulpcore-client 67 | if [[ "$TEST" = 'bindings' ]]; then 68 | ./generate.sh pulpcore ruby 0 69 | cd pulpcore-client 70 | gem build pulpcore_client.gemspec 71 | gem install --both ./pulpcore_client-0.gem 72 | fi 73 | cd $REPO_ROOT 74 | 75 | if [[ "$TEST" = 'bindings' ]]; then 76 | python $REPO_ROOT/.ci/assets/bindings/test_bindings.py 77 | fi 78 | 79 | if [[ "$TEST" = 'bindings' ]]; then 80 | if [ ! -f $REPO_ROOT/.ci/assets/bindings/test_bindings.rb ]; then 81 | exit 82 | else 83 | ruby $REPO_ROOT/.ci/assets/bindings/test_bindings.rb 84 | exit 85 | fi 86 | fi 87 | 88 | cat unittest_requirements.txt | cmd_stdin_prefix bash -c "cat > /tmp/unittest_requirements.txt" 89 | cmd_prefix pip3 install -r /tmp/unittest_requirements.txt 90 | 91 | # check for any uncommitted migrations 92 | echo "Checking for uncommitted migrations..." 93 | cmd_prefix bash -c "django-admin makemigrations --check --dry-run" 94 | 95 | # Run unit tests. 96 | cmd_prefix bash -c "PULP_DATABASES__default__USER=postgres django-admin test --noinput /usr/local/lib/python3.8/site-packages/pulp_python/tests/unit/" 97 | 98 | # Run functional tests 99 | export PYTHONPATH=$REPO_ROOT:$REPO_ROOT/../pulpcore${PYTHONPATH:+:${PYTHONPATH}} 100 | 101 | 102 | 103 | if [[ "$TEST" == "performance" ]]; then 104 | if [[ -z ${PERFORMANCE_TEST+x} ]]; then 105 | pytest -vv -r sx --color=yes --pyargs --capture=no --durations=0 pulp_python.tests.performance 106 | else 107 | pytest -vv -r sx --color=yes --pyargs --capture=no --durations=0 pulp_python.tests.performance.test_$PERFORMANCE_TEST 108 | fi 109 | exit 110 | fi 111 | 112 | if [ -f $FUNC_TEST_SCRIPT ]; then 113 | source $FUNC_TEST_SCRIPT 114 | else 115 | pytest -v -r sx --color=yes --pyargs pulp_python.tests.functional 116 | fi 117 | 118 | if [ -f $POST_SCRIPT ]; then 119 | source $POST_SCRIPT 120 | fi 121 | -------------------------------------------------------------------------------- /pulp_python/app/tasks/publish.py: -------------------------------------------------------------------------------- 1 | from gettext import gettext as _ 2 | import logging 3 | import os 4 | 5 | from django.core.files import File 6 | from packaging.utils import canonicalize_name 7 | 8 | from pulpcore.plugin import models 9 | 10 | from pulp_python.app import models as python_models 11 | from pulp_python.app.utils import write_simple_index, write_simple_detail 12 | 13 | 14 | log = logging.getLogger(__name__) 15 | 16 | 17 | def publish(repository_version_pk): 18 | """ 19 | Create a Publication based on a RepositoryVersion. 20 | 21 | Args: 22 | repository_version_pk (str): Create a Publication from this RepositoryVersion. 23 | 24 | """ 25 | repository_version = models.RepositoryVersion.objects.get(pk=repository_version_pk) 26 | 27 | log.info(_('Publishing: repository={repo}, version={version}').format( 28 | repo=repository_version.repository.name, 29 | version=repository_version.number, 30 | )) 31 | 32 | with python_models.PythonPublication.create(repository_version, pass_through=True) as pub: 33 | write_simple_api(pub) 34 | 35 | log.info(_('Publication: {pk} created').format(pk=pub.pk)) 36 | return pub 37 | 38 | 39 | def write_simple_api(publication): 40 | """ 41 | Write metadata for the simple API. 42 | 43 | Writes metadata mimicking the simple api of PyPI for all python packages 44 | in the repository version. 45 | 46 | https://wiki.python.org/moin/PyPISimple 47 | 48 | Args: 49 | publication (pulpcore.plugin.models.Publication): A publication to generate metadata for 50 | 51 | """ 52 | simple_dir = 'simple/' 53 | os.mkdir(simple_dir) 54 | project_names = ( 55 | python_models.PythonPackageContent.objects.filter( 56 | pk__in=publication.repository_version.content 57 | ) 58 | .order_by('name') 59 | .values_list('name', flat=True) 60 | .distinct() 61 | ) 62 | 63 | # write the root index, which lists all of the projects for which there is a package available 64 | index_path = '{simple_dir}index.html'.format(simple_dir=simple_dir) 65 | with open(index_path, 'w') as index: 66 | index.write(write_simple_index(project_names)) 67 | 68 | index_metadata = models.PublishedMetadata.create_from_file( 69 | relative_path=index_path, 70 | publication=publication, 71 | file=File(open(index_path, 'rb')) 72 | ) 73 | index_metadata.save() 74 | 75 | if len(project_names) == 0: 76 | return 77 | 78 | packages = python_models.PythonPackageContent.objects.filter( 79 | pk__in=publication.repository_version.content 80 | ) 81 | releases = packages.order_by("name").values("name", "filename", "sha256") 82 | 83 | ind = 0 84 | current_name = project_names[ind] 85 | package_releases = [] 86 | for release in releases.iterator(): 87 | if release['name'] != current_name: 88 | write_project_page( 89 | name=canonicalize_name(current_name), 90 | simple_dir=simple_dir, 91 | package_releases=package_releases, 92 | publication=publication 93 | ) 94 | package_releases = [] 95 | ind += 1 96 | current_name = project_names[ind] 97 | relative_path = release['filename'] 98 | path = f"../../{relative_path}" 99 | checksum = release['sha256'] 100 | package_releases.append((relative_path, path, checksum)) 101 | # Write the final project's page 102 | write_project_page( 103 | name=canonicalize_name(current_name), 104 | simple_dir=simple_dir, 105 | package_releases=package_releases, 106 | publication=publication 107 | ) 108 | 109 | 110 | def write_project_page(name, simple_dir, package_releases, publication): 111 | """Writes a project's simple page.""" 112 | project_dir = f'{simple_dir}{name}/' 113 | os.mkdir(project_dir) 114 | metadata_relative_path = f'{project_dir}index.html' 115 | 116 | with open(metadata_relative_path, 'w') as simple_metadata: 117 | simple_metadata.write(write_simple_detail(name, package_releases)) 118 | 119 | project_metadata = models.PublishedMetadata.create_from_file( 120 | relative_path=metadata_relative_path, 121 | publication=publication, 122 | file=File(open(metadata_relative_path, 'rb')) 123 | ) 124 | project_metadata.save() # change to bulk create when multi-table supported 125 | -------------------------------------------------------------------------------- /pulp_python/tests/functional/api/test_download_content.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | """Tests that verify download of content served by Pulp.""" 3 | import hashlib 4 | import unittest 5 | from random import choice 6 | from urllib.parse import urljoin 7 | 8 | from pulp_smash import utils 9 | from pulp_smash.pulp3.utils import ( 10 | download_content_unit, 11 | get_content_summary, 12 | ) 13 | from pulp_python.tests.functional.constants import ( 14 | PYTHON_FIXTURE_URL, 15 | PYTHON_LG_FIXTURE_SUMMARY, 16 | PYTHON_LG_PROJECT_SPECIFIER, 17 | ) 18 | from pulp_python.tests.functional.utils import ( 19 | cfg, 20 | get_python_content_paths, 21 | TestCaseUsingBindings, 22 | TestHelpersMixin, 23 | ) 24 | from pulp_python.tests.functional.utils import set_up_module as setUpModule # noqa:F401 25 | 26 | 27 | class DownloadContentTestCase(TestCaseUsingBindings, TestHelpersMixin): 28 | """Verify whether content served by pulp can be downloaded.""" 29 | 30 | def test_all(self): 31 | """Verify whether content served by pulp can be downloaded. 32 | 33 | The process of publishing content is more involved in Pulp 3 than it 34 | was under Pulp 2. Given a repository, the process is as follows: 35 | 36 | 1. Create a publication from the repository. (The latest repository 37 | version is selected if no version is specified.) A publication is a 38 | repository version plus metadata. 39 | 2. Create a distribution from the publication. The distribution defines 40 | at which URLs a publication is available, e.g. 41 | ``http://example.com/content/foo/`` and 42 | ``http://example.com/content/bar/``. 43 | 44 | Do the following: 45 | 46 | 1. Create, populate, publish, and distribute a repository. 47 | 2. Select a random content unit in the distribution. Download that 48 | content unit from Pulp, and verify that the content unit has the 49 | same checksum when fetched directly from Pulp-Fixtures. 50 | 51 | This test targets the following issues: 52 | 53 | * `Pulp #2895 `_ 54 | * `Pulp Smash #872 `_ 55 | """ 56 | remote = self._create_remote() 57 | repo = self._create_repo_and_sync_with_remote(remote) 58 | pub = self._create_publication(repo) 59 | distro = self._create_distribution_from_publication(pub) 60 | # Pick a content unit (of each type), and download it from both Pulp Fixtures… 61 | unit_paths = [ 62 | choice(paths) for paths in get_python_content_paths(repo.to_dict()).values() 63 | ] 64 | fixtures_hashes = [ 65 | hashlib.sha256( 66 | utils.http_get( 67 | urljoin(urljoin(PYTHON_FIXTURE_URL, "packages/"), unit_path[0]) 68 | ) 69 | ).hexdigest() 70 | for unit_path in unit_paths 71 | ] 72 | 73 | # …and Pulp. 74 | pulp_hashes = [] 75 | for unit_path in unit_paths: 76 | content = download_content_unit(cfg, distro.to_dict(), unit_path[1]) 77 | pulp_hashes.append(hashlib.sha256(content).hexdigest()) 78 | 79 | self.assertEqual(fixtures_hashes, pulp_hashes) 80 | 81 | 82 | class PublishPyPiJSON(TestCaseUsingBindings, TestHelpersMixin): 83 | """Test whether a distributed Python repository has a PyPi json endpoint 84 | a.k.a Can be consumed by another Pulp instance 85 | 86 | Test targets the following issue: 87 | 88 | * `Pulp #2886 `_ 89 | """ 90 | 91 | @unittest.skip("Content can not be synced without https") 92 | def test_basic_pulp_to_pulp_sync(self): 93 | """ 94 | This test checks that the JSON endpoint is setup correctly to allow one Pulp instance 95 | to perform a basic sync from another Pulp instance 96 | """ 97 | body = {"includes": PYTHON_LG_PROJECT_SPECIFIER, "prereleases": True} 98 | remote = self._create_remote(**body) 99 | repo = self._create_repo_and_sync_with_remote(remote) 100 | pub = self._create_publication(repo) 101 | distro = self._create_distribution_from_publication(pub) 102 | url_fragments = [ 103 | cfg.get_content_host_base_url(), 104 | "pulp/content", 105 | distro.base_path, 106 | "" 107 | ] 108 | unit_url = "/".join(url_fragments) 109 | 110 | body["url"] = unit_url 111 | remote = self._create_remote(**body) 112 | repo2 = self._create_repo_and_sync_with_remote(remote) 113 | self.assertEqual(get_content_summary(repo2.to_dict()), PYTHON_LG_FIXTURE_SUMMARY) 114 | -------------------------------------------------------------------------------- /.github/workflows/scripts/before_install.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # WARNING: DO NOT EDIT! 4 | # 5 | # This file was generated by plugin_template, and is managed by it. Please use 6 | # './plugin-template --github pulp_python' to update this file. 7 | # 8 | # For more info visit https://github.com/pulp/plugin_template 9 | 10 | # make sure this script runs at the repo root 11 | cd "$(dirname "$(realpath -e "$0")")"/../../.. 12 | 13 | set -mveuo pipefail 14 | 15 | if [ "${GITHUB_REF##refs/heads/}" = "${GITHUB_REF}" ] 16 | then 17 | BRANCH_BUILD=0 18 | else 19 | BRANCH_BUILD=1 20 | BRANCH="${GITHUB_REF##refs/heads/}" 21 | fi 22 | if [ "${GITHUB_REF##refs/tags/}" = "${GITHUB_REF}" ] 23 | then 24 | TAG_BUILD=0 25 | else 26 | TAG_BUILD=1 27 | BRANCH="${GITHUB_REF##refs/tags/}" 28 | fi 29 | 30 | if [[ "$TEST" == "upgrade" ]]; then 31 | git checkout -b ci_upgrade_test 32 | cp -R .github /tmp/.github 33 | cp -R .ci /tmp/.ci 34 | git checkout $FROM_PULP_PYTHON_BRANCH 35 | rm -rf .ci .github 36 | cp -R /tmp/.github . 37 | cp -R /tmp/.ci . 38 | # Pin deps 39 | sed -i "s/~/=/g" requirements.txt 40 | fi 41 | 42 | if [[ "$TEST" == "plugin-from-pypi" ]]; then 43 | COMPONENT_VERSION=$(http https://pypi.org/pypi/pulp-python/json | jq -r '.info.version') 44 | else 45 | COMPONENT_VERSION=$(sed -ne "s/\s*version=['\"]\(.*\)['\"][\s,]*/\1/p" setup.py) 46 | fi 47 | mkdir .ci/ansible/vars || true 48 | echo "---" > .ci/ansible/vars/main.yaml 49 | echo "legacy_component_name: pulp_python" >> .ci/ansible/vars/main.yaml 50 | echo "component_name: python" >> .ci/ansible/vars/main.yaml 51 | echo "component_version: '${COMPONENT_VERSION}'" >> .ci/ansible/vars/main.yaml 52 | 53 | export PRE_BEFORE_INSTALL=$PWD/.github/workflows/scripts/pre_before_install.sh 54 | export POST_BEFORE_INSTALL=$PWD/.github/workflows/scripts/post_before_install.sh 55 | 56 | COMMIT_MSG=$(git log --format=%B --no-merges -1) 57 | export COMMIT_MSG 58 | 59 | if [ -f $PRE_BEFORE_INSTALL ]; then 60 | source $PRE_BEFORE_INSTALL 61 | fi 62 | 63 | if [[ -n $(echo -e $COMMIT_MSG | grep -P "Required PR:.*" | grep -v "https") ]]; then 64 | echo "Invalid Required PR link detected in commit message. Please use the full https url." 65 | exit 1 66 | fi 67 | 68 | if [ "$GITHUB_EVENT_NAME" = "pull_request" ] || [ "${BRANCH_BUILD}" = "1" -a "${BRANCH}" != "master" ] 69 | then 70 | export PULPCORE_PR_NUMBER=$(echo $COMMIT_MSG | grep -oP 'Required\ PR:\ https\:\/\/github\.com\/pulp\/pulpcore\/pull\/(\d+)' | awk -F'/' '{print $7}') 71 | export PULP_SMASH_PR_NUMBER=$(echo $COMMIT_MSG | grep -oP 'Required\ PR:\ https\:\/\/github\.com\/pulp\/pulp-smash\/pull\/(\d+)' | awk -F'/' '{print $7}') 72 | export PULP_OPENAPI_GENERATOR_PR_NUMBER=$(echo $COMMIT_MSG | grep -oP 'Required\ PR:\ https\:\/\/github\.com\/pulp\/pulp-openapi-generator\/pull\/(\d+)' | awk -F'/' '{print $7}') 73 | export PULP_CLI_PR_NUMBER=$(echo $COMMIT_MSG | grep -oP 'Required\ PR:\ https\:\/\/github\.com\/pulp\/pulp-cli\/pull\/(\d+)' | awk -F'/' '{print $7}') 74 | echo $COMMIT_MSG | sed -n -e 's/.*CI Base Image:\s*\([-_/[:alnum:]]*:[-_[:alnum:]]*\).*/ci_base: "\1"/p' >> .ci/ansible/vars/main.yaml 75 | else 76 | export PULPCORE_PR_NUMBER= 77 | export PULP_SMASH_PR_NUMBER= 78 | export PULP_OPENAPI_GENERATOR_PR_NUMBER= 79 | export PULP_CLI_PR_NUMBER= 80 | export CI_BASE_IMAGE= 81 | fi 82 | 83 | 84 | cd .. 85 | 86 | 87 | git clone --depth=1 https://github.com/pulp/pulp-smash.git 88 | 89 | if [ -n "$PULP_SMASH_PR_NUMBER" ]; then 90 | cd pulp-smash 91 | git fetch --depth=1 origin pull/$PULP_SMASH_PR_NUMBER/head:$PULP_SMASH_PR_NUMBER 92 | git checkout $PULP_SMASH_PR_NUMBER 93 | cd .. 94 | fi 95 | 96 | pip install --upgrade --force-reinstall ./pulp-smash 97 | 98 | 99 | git clone --depth=1 https://github.com/pulp/pulp-openapi-generator.git 100 | if [ -n "$PULP_OPENAPI_GENERATOR_PR_NUMBER" ]; then 101 | cd pulp-openapi-generator 102 | git fetch origin pull/$PULP_OPENAPI_GENERATOR_PR_NUMBER/head:$PULP_OPENAPI_GENERATOR_PR_NUMBER 103 | git checkout $PULP_OPENAPI_GENERATOR_PR_NUMBER 104 | cd .. 105 | fi 106 | 107 | 108 | 109 | git clone --depth=1 https://github.com/pulp/pulpcore.git --branch master 110 | 111 | cd pulpcore 112 | if [ -n "$PULPCORE_PR_NUMBER" ]; then 113 | git fetch --depth=1 origin pull/$PULPCORE_PR_NUMBER/head:$PULPCORE_PR_NUMBER 114 | git checkout $PULPCORE_PR_NUMBER 115 | fi 116 | cd .. 117 | 118 | 119 | 120 | 121 | # Intall requirements for ansible playbooks 122 | pip install docker netaddr boto3 ansible 123 | 124 | for i in {1..3} 125 | do 126 | ansible-galaxy collection install amazon.aws && s=0 && break || s=$? && sleep 3 127 | done 128 | if [[ $s -gt 0 ]] 129 | then 130 | echo "Failed to install amazon.aws" 131 | exit $s 132 | fi 133 | 134 | cd pulp_python 135 | 136 | if [ -f $POST_BEFORE_INSTALL ]; then 137 | source $POST_BEFORE_INSTALL 138 | fi 139 | -------------------------------------------------------------------------------- /pulp_python/app/tasks/upload.py: -------------------------------------------------------------------------------- 1 | import pkginfo 2 | import shutil 3 | import tempfile 4 | import time 5 | 6 | from datetime import datetime, timezone 7 | from django.db import transaction 8 | from django.contrib.sessions.models import Session 9 | from django.core.files.storage import default_storage as storage 10 | from pulpcore.plugin.models import Artifact, CreatedResource, ContentArtifact 11 | 12 | from pulp_python.app.models import PythonPackageContent, PythonRepository 13 | from pulp_python.app.utils import parse_project_metadata 14 | 15 | 16 | DIST_EXTENSIONS = { 17 | ".whl": "bdist_wheel", 18 | ".exe": "bdist_wininst", 19 | ".egg": "bdist_egg", 20 | ".tar.bz2": "sdist", 21 | ".tar.gz": "sdist", 22 | ".zip": "sdist", 23 | } 24 | 25 | DIST_TYPES = { 26 | "bdist_wheel": pkginfo.Wheel, 27 | "bdist_wininst": pkginfo.Distribution, 28 | "bdist_egg": pkginfo.BDist, 29 | "sdist": pkginfo.SDist, 30 | } 31 | 32 | 33 | def upload(artifact_sha256, filename, repository_pk=None): 34 | """ 35 | Uploads a Python Package to Pulp 36 | 37 | Args: 38 | artifact_sha256: the sha256 of the artifact in Pulp to create a package from 39 | filename: the full filename of the package to create 40 | repository_pk: the optional pk of the repository to add the content to 41 | """ 42 | pre_check = PythonPackageContent.objects.filter(sha256=artifact_sha256) 43 | content_to_add = pre_check or create_content(artifact_sha256, filename) 44 | if repository_pk: 45 | repository = PythonRepository.objects.get(pk=repository_pk) 46 | with repository.new_version() as new_version: 47 | new_version.add_content(content_to_add) 48 | 49 | 50 | def upload_group(session_pk, repository_pk=None): 51 | """ 52 | Uploads a Python Package to Pulp 53 | 54 | Args: 55 | session_pk: the session that has the artifacts to upload 56 | repository_pk: optional repository to add Content to 57 | """ 58 | s_query = Session.objects.select_for_update().filter(pk=session_pk) 59 | while True: 60 | with transaction.atomic(): 61 | session_data = s_query.first().get_decoded() 62 | now = datetime.now(tz=timezone.utc) 63 | try: 64 | start_time = datetime.fromisoformat(session_data['start']) 65 | except AttributeError: 66 | # TODO: Remove this once Python 3.7+ project 67 | from dateutil.parser import parse 68 | start_time = parse(session_data['start']) 69 | if now >= start_time: 70 | content_to_add = PythonPackageContent.objects.none() 71 | for artifact_sha256, filename in session_data['artifacts']: 72 | pre_check = PythonPackageContent.objects.filter(sha256=artifact_sha256) 73 | content_to_add |= pre_check or create_content(artifact_sha256, filename) 74 | 75 | if repository_pk: 76 | repository = PythonRepository.objects.get(pk=repository_pk) 77 | with repository.new_version() as new_version: 78 | new_version.add_content(content_to_add) 79 | return 80 | else: 81 | sleep_time = start_time - now 82 | time.sleep(sleep_time.seconds) 83 | 84 | 85 | def create_content(artifact_sha256, filename): 86 | """ 87 | Creates PythonPackageContent from artifact. 88 | 89 | Args: 90 | artifact_sha256: validated artifact 91 | filename: file name 92 | Returns: 93 | queryset of the new created content 94 | """ 95 | # iterate through extensions since splitext does not support things like .tar.gz 96 | extensions = list(DIST_EXTENSIONS.keys()) 97 | pkg_type_index = [filename.endswith(ext) for ext in extensions].index(True) 98 | packagetype = DIST_EXTENSIONS[extensions[pkg_type_index]] 99 | # Copy file to a temp directory under the user provided filename, we do this 100 | # because pkginfo validates that the filename has a valid extension before 101 | # reading it 102 | artifact = Artifact.objects.get(sha256=artifact_sha256) 103 | artifact_file = storage.open(artifact.file.name) 104 | with tempfile.NamedTemporaryFile('wb', suffix=filename) as temp_file: 105 | shutil.copyfileobj(artifact_file, temp_file) 106 | temp_file.flush() 107 | metadata = DIST_TYPES[packagetype](temp_file.name) 108 | metadata.packagetype = packagetype 109 | 110 | data = parse_project_metadata(vars(metadata)) 111 | data['packagetype'] = metadata.packagetype 112 | data['version'] = metadata.version 113 | data['filename'] = filename 114 | data['sha256'] = artifact.sha256 115 | 116 | @transaction.atomic() 117 | def create(): 118 | content = PythonPackageContent.objects.create(**data) 119 | ContentArtifact.objects.create( 120 | artifact=artifact, content=content, relative_path=filename 121 | ) 122 | return content 123 | 124 | new_content = create() 125 | resource = CreatedResource(content_object=new_content) 126 | resource.save() 127 | 128 | return PythonPackageContent.objects.filter(pk=new_content.pk) 129 | -------------------------------------------------------------------------------- /pulp_python/app/migrations/0001_initial.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.3 on 2020-06-18 13:55 2 | 3 | import django.contrib.postgres.fields.jsonb 4 | from django.db import migrations, models 5 | import django.db.models.deletion 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | initial = True 11 | 12 | dependencies = [ 13 | ('core', '0032_export_to_chunks'), 14 | ] 15 | 16 | operations = [ 17 | migrations.CreateModel( 18 | name='PythonPublication', 19 | fields=[ 20 | ('publication_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, related_name='python_pythonpublication', serialize=False, to='core.Publication')), 21 | ], 22 | options={ 23 | 'default_related_name': '%(app_label)s_%(model_name)s', 24 | }, 25 | bases=('core.publication',), 26 | ), 27 | migrations.CreateModel( 28 | name='PythonRemote', 29 | fields=[ 30 | ('remote_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, related_name='python_pythonremote', serialize=False, to='core.Remote')), 31 | ('prereleases', models.BooleanField(default=False)), 32 | ('includes', django.contrib.postgres.fields.jsonb.JSONField(default=list)), 33 | ('excludes', django.contrib.postgres.fields.jsonb.JSONField(default=list)), 34 | ], 35 | options={ 36 | 'default_related_name': '%(app_label)s_%(model_name)s', 37 | }, 38 | bases=('core.remote',), 39 | ), 40 | migrations.CreateModel( 41 | name='PythonRepository', 42 | fields=[ 43 | ('repository_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, related_name='python_pythonrepository', serialize=False, to='core.Repository')), 44 | ], 45 | options={ 46 | 'default_related_name': '%(app_label)s_%(model_name)s', 47 | }, 48 | bases=('core.repository',), 49 | ), 50 | migrations.CreateModel( 51 | name='PythonPackageContent', 52 | fields=[ 53 | ('content_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, related_name='python_pythonpackagecontent', serialize=False, to='core.Content')), 54 | ('filename', models.TextField(db_index=True, unique=True)), 55 | ('packagetype', models.TextField(choices=[('bdist_dmg', 'bdist_dmg'), ('bdist_dumb', 'bdist_dumb'), ('bdist_egg', 'bdist_egg'), ('bdist_msi', 'bdist_msi'), ('bdist_rpm', 'bdist_rpm'), ('bdist_wheel', 'bdist_wheel'), ('bdist_wininst', 'bdist_wininst'), ('sdist', 'sdist')])), 56 | ('name', models.TextField()), 57 | ('version', models.TextField()), 58 | ('metadata_version', models.TextField()), 59 | ('summary', models.TextField()), 60 | ('description', models.TextField()), 61 | ('keywords', models.TextField()), 62 | ('home_page', models.TextField()), 63 | ('download_url', models.TextField()), 64 | ('author', models.TextField()), 65 | ('author_email', models.TextField()), 66 | ('maintainer', models.TextField()), 67 | ('maintainer_email', models.TextField()), 68 | ('license', models.TextField()), 69 | ('requires_python', models.TextField()), 70 | ('project_url', models.TextField()), 71 | ('platform', models.TextField()), 72 | ('supported_platform', models.TextField()), 73 | ('requires_dist', django.contrib.postgres.fields.jsonb.JSONField(default=list)), 74 | ('provides_dist', django.contrib.postgres.fields.jsonb.JSONField(default=list)), 75 | ('obsoletes_dist', django.contrib.postgres.fields.jsonb.JSONField(default=list)), 76 | ('requires_external', django.contrib.postgres.fields.jsonb.JSONField(default=list)), 77 | ('classifiers', django.contrib.postgres.fields.jsonb.JSONField(default=list)), 78 | ], 79 | options={ 80 | 'default_related_name': '%(app_label)s_%(model_name)s', 81 | 'unique_together': {('filename',)}, 82 | }, 83 | bases=('core.content',), 84 | ), 85 | migrations.CreateModel( 86 | name='PythonDistribution', 87 | fields=[ 88 | ('basedistribution_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, related_name='python_pythondistribution', serialize=False, to='core.BaseDistribution')), 89 | ('publication', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='python_pythondistribution', to='core.Publication')), 90 | ], 91 | options={ 92 | 'default_related_name': '%(app_label)s_%(model_name)s', 93 | }, 94 | bases=('core.basedistribution',), 95 | ), 96 | ] 97 | -------------------------------------------------------------------------------- /pulp_python/tests/functional/api/test_consume_content.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | """Tests that perform actions over content unit.""" 3 | from pulp_smash import cli 4 | from pulp_smash.pulp3.bindings import monitor_task 5 | from pulp_smash.pulp3.utils import delete_orphans, modify_repo 6 | 7 | from pulp_python.tests.functional.constants import ( 8 | PYTHON_FIXTURE_URL, 9 | PYTHON_FIXTURES_PACKAGES, 10 | PYTHON_FIXTURES_FILENAMES, 11 | PYTHON_LIST_PROJECT_SPECIFIER, 12 | PYPI_URL, 13 | ) 14 | 15 | from pulp_python.tests.functional.utils import ( 16 | cfg, 17 | gen_artifact, 18 | gen_python_content_attrs, 19 | TestCaseUsingBindings, 20 | TestHelpersMixin, 21 | ) 22 | from pulp_python.tests.functional.utils import set_up_module as setUpModule # noqa:F401 23 | from urllib.parse import urljoin, urlsplit 24 | 25 | from pulp_smash.utils import http_get 26 | 27 | 28 | class PipInstallContentTestCase(TestCaseUsingBindings, TestHelpersMixin): 29 | """ 30 | Verify whether content served by Pulp can be consumed through pip install. 31 | Workflows tested are: 32 | 1) create repo -> upload package artifact -> create content -> add content to repo -> publish 33 | repo -> distribute publication -> consume through pip 34 | 2) create repo -> sync with remote -> publish repo -> distribute publication -> consume through 35 | pip 36 | """ 37 | 38 | @classmethod 39 | def setUpClass(cls): 40 | """ 41 | Check if packages to install through tests are already installed 42 | """ 43 | super().setUpClass() 44 | cls.cli_client = cli.Client(cfg) 45 | cls.PACKAGES = PYTHON_FIXTURES_PACKAGES 46 | cls.PACKAGES_URLS = [ 47 | urljoin(urljoin(PYTHON_FIXTURE_URL, "packages/"), filename) 48 | for filename in PYTHON_FIXTURES_FILENAMES 49 | ] 50 | cls.PACKAGES_FILES = [{"file": http_get(file)} for file in cls.PACKAGES_URLS] 51 | delete_orphans() 52 | for pkg in cls.PACKAGES: 53 | cls.assertFalse( 54 | cls, 55 | cls.check_install(cls.cli_client, pkg), 56 | "{} is already installed".format(pkg), 57 | ) 58 | 59 | def test_workflow_01(self): 60 | """ 61 | Verify workflow 1 62 | """ 63 | created_contents = [] 64 | for pkg, filename in zip(self.PACKAGES_URLS, PYTHON_FIXTURES_FILENAMES): 65 | content_response = self.content_api.create( 66 | **gen_python_content_attrs(gen_artifact(pkg), filename) 67 | ) 68 | created_contents.extend(monitor_task(content_response.task).created_resources) 69 | created_contents = [self.content_api.read(href).to_dict() for href in created_contents] 70 | 71 | repo = self._create_repository() 72 | # Add content 73 | modify_repo(cfg, repo.to_dict(), add_units=created_contents) 74 | repo = self.repo_api.read(repo.pulp_href) 75 | pub = self._create_publication(repo) 76 | distro = self._create_distribution_from_publication(pub) 77 | 78 | self.addCleanup(delete_orphans, cfg) 79 | self.check_consume(distro.to_dict()) 80 | 81 | def test_workflow_02(self): 82 | """ 83 | Verify workflow 2 84 | 85 | Do the following: 86 | 87 | 1. Create, populate, publish, and distribute a repository. 88 | 2. Pip install a package from the pulp repository. 89 | 3. Check pip install was successful. 90 | 91 | This test targets the following issues: 92 | * `Pulp #4682 `_ 93 | * `Pulp #4677 `_ 94 | """ 95 | remote = self._create_remote(includes=PYTHON_LIST_PROJECT_SPECIFIER) 96 | repo = self._create_repo_and_sync_with_remote(remote) 97 | pub = self._create_publication(repo) 98 | distro = self._create_distribution_from_publication(pub) 99 | self.check_consume(distro.to_dict()) 100 | 101 | def check_consume(self, distribution): 102 | """Tests that pip packages hosted in a distribution can be consumed""" 103 | host_base_url = cfg.get_content_host_base_url() 104 | url = "".join( 105 | [host_base_url, "/pulp/content/", distribution["base_path"], "/simple/"] 106 | ) 107 | for pkg in self.PACKAGES: 108 | out = self.install(self.cli_client, pkg, host=url) 109 | self.assertTrue(self.check_install(self.cli_client, pkg), out) 110 | self.addCleanup(self.uninstall, self.cli_client, pkg) 111 | 112 | @staticmethod 113 | def check_install(cli_client, package): 114 | """Returns true if python package is installed, false otherwise""" 115 | return cli_client.run(("pip", "list")).stdout.find(package) != -1 116 | 117 | @staticmethod 118 | def install(cli_client, package, host=PYPI_URL): 119 | """Installs a pip package from the host url""" 120 | return cli_client.run( 121 | ( 122 | "pip", 123 | "install", 124 | "--no-deps", 125 | "--trusted-host", 126 | urlsplit(host).hostname, 127 | "-i", 128 | host, 129 | package, 130 | ) 131 | ).stdout 132 | 133 | @staticmethod 134 | def uninstall(cli_client, package): 135 | """ 136 | Uninstalls a pip package and returns the version number 137 | Uninstall Message format: "Found existing installation: package X.X.X ..." 138 | """ 139 | return cli_client.run(("pip", "uninstall", package, "-y")).stdout.split()[4] 140 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = -W -n 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = _build 9 | STATIC_BUILD_DIR = _static 10 | DIAGRAM_BUILD_DIR = _diagrams 11 | DIAGRAM_SOURCE_DIR = diagrams_src 12 | PULP_URL = "http://localhost:24817" 13 | 14 | # Internal variables. 15 | PAPEROPT_a4 = -D latex_paper_size=a4 16 | PAPEROPT_letter = -D latex_paper_size=letter 17 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 18 | # the i18n builder cannot share the environment and doctrees with the others 19 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 20 | 21 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext 22 | 23 | help: 24 | @echo "Please use \`make ' where is one of" 25 | @echo " html to make standalone HTML files" 26 | @echo " dirhtml to make HTML files named index.html in directories" 27 | @echo " singlehtml to make a single large HTML file" 28 | @echo " pickle to make pickle files" 29 | @echo " json to make JSON files" 30 | @echo " htmlhelp to make HTML files and a HTML help project" 31 | @echo " qthelp to make HTML files and a qthelp project" 32 | @echo " devhelp to make HTML files and a Devhelp project" 33 | @echo " epub to make an epub" 34 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 35 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 36 | @echo " text to make text files" 37 | @echo " man to make manual pages" 38 | @echo " texinfo to make Texinfo files" 39 | @echo " info to make Texinfo files and run them through makeinfo" 40 | @echo " gettext to make PO message catalogs" 41 | @echo " changes to make an overview of all changed/added/deprecated items" 42 | @echo " linkcheck to check all external links for integrity" 43 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 44 | 45 | clean: 46 | -rm -rf $(BUILDDIR)/* 47 | -rm -rf $(DIAGRAM_BUILD_DIR)/* 48 | 49 | diagrams: 50 | ifneq ($(wildcard $(DIAGRAM_SOURCE_DIR)), ) 51 | mkdir -p $(DIAGRAM_BUILD_DIR) 52 | python3 -m plantuml $(DIAGRAM_SOURCE_DIR)/*.dot 53 | # cp + rm = mv workaround https://pulp.plan.io/issues/4791#note-3 54 | cp $(DIAGRAM_SOURCE_DIR)/*.png $(DIAGRAM_BUILD_DIR)/ 55 | rm $(DIAGRAM_SOURCE_DIR)/*.png 56 | else 57 | @echo "Did not find $(DIAGRAM_SOURCE_DIR)." 58 | endif 59 | 60 | html: 61 | mkdir -p $(STATIC_BUILD_DIR) 62 | curl --fail -o $(STATIC_BUILD_DIR)/api.json "$(PULP_URL)/pulp/api/v3/docs/api.json?plugin=pulp_python&include_html=1" 63 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 64 | @echo 65 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 66 | 67 | dirhtml: 68 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 69 | @echo 70 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 71 | 72 | singlehtml: 73 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 74 | @echo 75 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 76 | 77 | pickle: 78 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 79 | @echo 80 | @echo "Build finished; now you can process the pickle files." 81 | 82 | json: 83 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 84 | @echo 85 | @echo "Build finished; now you can process the JSON files." 86 | 87 | htmlhelp: 88 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 89 | @echo 90 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 91 | ".hhp project file in $(BUILDDIR)/htmlhelp." 92 | 93 | epub: 94 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 95 | @echo 96 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 97 | 98 | latex: 99 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 100 | @echo 101 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 102 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 103 | "(use \`make latexpdf' here to do that automatically)." 104 | 105 | latexpdf: 106 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 107 | @echo "Running LaTeX files through pdflatex..." 108 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 109 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 110 | 111 | text: 112 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 113 | @echo 114 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 115 | 116 | man: 117 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 118 | @echo 119 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 120 | 121 | texinfo: 122 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 123 | @echo 124 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 125 | @echo "Run \`make' in that directory to run these through makeinfo" \ 126 | "(use \`make info' here to do that automatically)." 127 | 128 | info: 129 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 130 | @echo "Running Texinfo files through makeinfo..." 131 | make -C $(BUILDDIR)/texinfo info 132 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 133 | 134 | gettext: 135 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 136 | @echo 137 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 138 | 139 | changes: 140 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 141 | @echo 142 | @echo "The overview file is in $(BUILDDIR)/changes." 143 | 144 | linkcheck: 145 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 146 | @echo 147 | @echo "Link check complete; look for any errors in the above output " \ 148 | "or in $(BUILDDIR)/linkcheck/output.txt." 149 | 150 | doctest: 151 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 152 | @echo "Testing of doctests in the sources finished, look at the " \ 153 | "results in $(BUILDDIR)/doctest/output.txt." 154 | -------------------------------------------------------------------------------- /docs/workflows/sync.rst: -------------------------------------------------------------------------------- 1 | .. _sync-workflow: 2 | 3 | Synchronize a Repository 4 | ======================== 5 | 6 | Users can populate their repositories with content from an external source like PyPI by syncing 7 | their repository. 8 | 9 | .. _versioned-repo-created: 10 | 11 | Create a Repository 12 | ------------------- 13 | 14 | .. literalinclude:: ../_scripts/repo.sh 15 | :language: bash 16 | 17 | Repository Create Response:: 18 | 19 | { 20 | "pulp_href": "/pulp/api/v3/repositories/python/python/8fbb24ee-dc91-44f4-a6ee-beec60aa542d/", 21 | "pulp_created": "2021-03-09T04:11:54.347921Z", 22 | "versions_href": "/pulp/api/v3/repositories/python/python/8fbb24ee-dc91-44f4-a6ee-beec60aa542d/versions/", 23 | "pulp_labels": {}, 24 | "latest_version_href": "/pulp/api/v3/repositories/python/python/8fbb24ee-dc91-44f4-a6ee-beec60aa542d/versions/0/", 25 | "name": "foo", 26 | "description": null, 27 | "remote": null 28 | } 29 | 30 | Reference (pulpcore): `Repository API Usage 31 | `_ 32 | 33 | .. _create-remote: 34 | 35 | Create a Remote 36 | --------------- 37 | 38 | Creating a remote object informs Pulp about an external content source. In this case, we will be 39 | using a fixture, but Python remotes can be anything that implements the PyPI API. This can be PyPI 40 | itself, a fixture, or even an instance of Pulp 2. 41 | 42 | .. literalinclude:: ../_scripts/remote.sh 43 | :language: bash 44 | 45 | Remote Create Response:: 46 | 47 | { 48 | "pulp_href": "/pulp/api/v3/remotes/python/python/a9bb3a02-c7d2-4b2e-9b66-050a6c9b7cb3/", 49 | "pulp_created": "2021-03-09T04:14:02.646835Z", 50 | "name": "bar", 51 | "url": "https://pypi.org/", 52 | "ca_cert": null, 53 | "client_cert": null, 54 | "tls_validation": true, 55 | "proxy_url": null, 56 | "pulp_labels": {}, 57 | "pulp_last_updated": "2021-03-09T04:14:02.646845Z", 58 | "download_concurrency": 10, 59 | "policy": "on_demand", 60 | "total_timeout": null, 61 | "connect_timeout": null, 62 | "sock_connect_timeout": null, 63 | "sock_read_timeout": null, 64 | "headers": null, 65 | "rate_limit": null, 66 | "includes": [ 67 | "shelf-reader" 68 | ], 69 | "excludes": [], 70 | "prereleases": true, 71 | } 72 | 73 | 74 | Reference: `Python Remote Usage <../restapi.html#tag/Remotes:-Python>`_ 75 | 76 | A More Complex Remote 77 | --------------------- 78 | 79 | If only the name of a project is specified, every distribution of every version of that project 80 | will be synced. You can use the version_specifier field to ensure only distributions you care 81 | about will be synced:: 82 | 83 | $ pulp python remote create \ 84 | --name 'complex-remote' \ 85 | --url 'https://pypi.org/' \ 86 | --includes '[ 87 | "django~=2.0,!=2.0.1", 88 | "pip-tools>=1.12,<=2.0", 89 | "scipy", 90 | "shelf-reader" 91 | ]' 92 | 93 | You can also use version specifiers to "exclude" certain versions of a project, like so:: 94 | 95 | $ pulp python remote create \ 96 | --name 'complex-remote' \ 97 | --url 'https://pypi.org/' \ 98 | --includes '[ 99 | "django", 100 | "scipy" 101 | ]' \ 102 | --excludes '[ 103 | "django~=1.0", 104 | "scipy" 105 | ]' 106 | 107 | You can also filter packages by their type, platform and amount synced through the "package_types", 108 | "exclude_platforms", and "keep_latest_packages" fields respectively, like so:: 109 | 110 | $ pulp python remote create \ 111 | --name 'complex-filters' \ 112 | --url 'https://pypi.org/' \ 113 | --includes '["django"]' \ 114 | --package-types '["sdist", "bdist-wheel"]' # only sync sdist and bdist-wheel package types \ 115 | --exclude-platforms '["windows"]' # exclude any packages built for windows \ 116 | --keep-latest-packages 5 # keep the five latest versions 117 | 118 | Reference: `Python Remote Usage <../restapi.html#tag/Remotes:-Python>`_ 119 | 120 | .. _mirror-workflow: 121 | 122 | Creating a remote to sync all of PyPI 123 | _____________________________________ 124 | 125 | A remote can be setup to sync all of PyPI by not specifying any included packages like so:: 126 | 127 | $ pulp python remote create \ 128 | --name 'PyPI-mirror' \ 129 | --url 'https://pypi.org/' \ 130 | --excludes '[ 131 | "django~=1.0", 132 | "scipy" 133 | ]' 134 | 135 | By not setting the "includes" field Pulp will ask PyPI for all of its available packages to sync, minus the ones from 136 | the excludes field. Default Python remotes are created with syncing policy "on_demand" because the most common 137 | Python remotes involve syncing with PyPI which requires terabytes of disk space. This can be changed by 138 | modifying the "policy" field. 139 | 140 | Sync repository foo with remote 141 | ------------------------------- 142 | 143 | Use the remote object to kick off a synchronize task by specifying the repository to 144 | sync with. You are telling pulp to fetch content from the remote and add to the repository. 145 | 146 | .. literalinclude:: ../_scripts/sync.sh 147 | :language: bash 148 | 149 | Repository Version Show Response (when complete):: 150 | 151 | { 152 | "pulp_href": "/pulp/api/v3/repositories/python/python/8fbb24ee-dc91-44f4-a6ee-beec60aa542d/versions/1/", 153 | "pulp_created": "2021-03-09T04:20:21.896132Z", 154 | "number": 1, 155 | "base_version": null, 156 | "content_summary": { 157 | "added": { 158 | "python.python": { 159 | "count": 2, 160 | "href": "/pulp/api/v3/content/python/packages/?repository_version_added=/pulp/api/v3/repositories/python/python/8fbb24ee-dc91-44f4-a6ee-beec60aa542d/versions/1/" 161 | } 162 | }, 163 | "removed": {}, 164 | "present": { 165 | "python.python": { 166 | "count": 2, 167 | "href": "/pulp/api/v3/content/python/packages/?repository_version=/pulp/api/v3/repositories/python/python/8fbb24ee-dc91-44f4-a6ee-beec60aa542d/versions/1/" 168 | } 169 | } 170 | } 171 | } 172 | 173 | 174 | 175 | 176 | Reference: `Python Sync Usage <../restapi.html#operation/repositories_python_python_sync>`_ 177 | 178 | Reference (pulpcore): `Repository Version Creation API Usage 179 | `_ 180 | -------------------------------------------------------------------------------- /.github/workflows/nightly.yml: -------------------------------------------------------------------------------- 1 | # WARNING: DO NOT EDIT! 2 | # 3 | # This file was generated by plugin_template, and is managed by it. Please use 4 | # './plugin-template --github pulp_python' to update this file. 5 | # 6 | # For more info visit https://github.com/pulp/plugin_template 7 | --- 8 | name: Pulp Nightly CI/CD 9 | on: 10 | schedule: 11 | # * is a special character in YAML so you have to quote this string 12 | # runs at 3:00 UTC daily 13 | - cron: '00 3 * * *' 14 | 15 | jobs: 16 | test: 17 | runs-on: ubuntu-latest 18 | 19 | strategy: 20 | fail-fast: false 21 | matrix: 22 | env: 23 | - TEST: pulp 24 | - TEST: docs 25 | - TEST: s3 26 | - TEST: generate-bindings 27 | 28 | steps: 29 | - uses: actions/checkout@v2 30 | with: 31 | # by default, it uses a depth of 1 32 | # this fetches all history so that we can read each commit 33 | fetch-depth: 0 34 | 35 | - uses: actions/setup-python@v2 36 | with: 37 | python-version: "3.7" 38 | 39 | - name: Install httpie 40 | run: | 41 | echo ::group::HTTPIE 42 | sudo apt-get update -yq 43 | sudo -E apt-get -yq --no-install-suggests --no-install-recommends install httpie 44 | echo ::endgroup:: 45 | echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/.ci/assets/httpie/" >> $GITHUB_ENV 46 | echo "TEST=${{ matrix.env.TEST }}" >> $GITHUB_ENV 47 | 48 | - name: Before Install 49 | run: .github/workflows/scripts/before_install.sh 50 | shell: bash 51 | 52 | - uses: ruby/setup-ruby@v1 53 | if: ${{ env.TEST == 'bindings' || env.TEST == 'generate-bindings' }} 54 | with: 55 | ruby-version: "2.6" 56 | 57 | - name: Install 58 | run: .github/workflows/scripts/install.sh 59 | env: 60 | PY_COLORS: '1' 61 | ANSIBLE_FORCE_COLOR: '1' 62 | shell: bash 63 | 64 | - name: Before Script 65 | run: | 66 | .github/workflows/scripts/before_script.sh 67 | 68 | - name: Setting secrets 69 | run: python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" 70 | env: 71 | SECRETS_CONTEXT: ${{ toJson(secrets) }} 72 | 73 | - name: Install Python client 74 | run: .github/workflows/scripts/install_python_client.sh 75 | 76 | - name: Install Ruby client 77 | if: ${{ env.TEST == 'bindings' || env.TEST == 'generate-bindings' }} 78 | run: .github/workflows/scripts/install_ruby_client.sh 79 | 80 | - name: Script 81 | run: .github/workflows/scripts/script.sh 82 | shell: bash 83 | 84 | - name: Upload python client packages 85 | if: ${{ env.TEST == 'bindings' || env.TEST == 'generate-bindings' }} 86 | uses: actions/upload-artifact@v2 87 | with: 88 | name: python-client.tar 89 | path: python-client.tar 90 | 91 | - name: Upload ruby client packages 92 | if: ${{ env.TEST == 'bindings' || env.TEST == 'generate-bindings' }} 93 | uses: actions/upload-artifact@v2 94 | with: 95 | name: ruby-client.tar 96 | path: ruby-client.tar 97 | 98 | - name: Upload built docs 99 | if: ${{ env.TEST == 'docs' }} 100 | uses: actions/upload-artifact@v2 101 | with: 102 | name: docs.tar 103 | path: docs/docs.tar 104 | 105 | publish: 106 | runs-on: ubuntu-latest 107 | needs: test 108 | 109 | env: 110 | TEST: publish 111 | 112 | steps: 113 | - uses: actions/checkout@v2 114 | with: 115 | # by default, it uses a depth of 1 116 | # this fetches all history so that we can read each commit 117 | fetch-depth: 0 118 | 119 | - uses: actions/setup-python@v2 120 | with: 121 | python-version: "3.7" 122 | 123 | - uses: actions/setup-ruby@v1 124 | with: 125 | ruby-version: "2.6" 126 | 127 | - name: Install httpie 128 | run: | 129 | echo ::group::HTTPIE 130 | sudo apt-get update -yq 131 | sudo -E apt-get -yq --no-install-suggests --no-install-recommends install httpie 132 | echo ::endgroup:: 133 | echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/.ci/assets/httpie/" >> $GITHUB_ENV 134 | 135 | - name: Install python dependencies 136 | run: | 137 | echo ::group::PYDEPS 138 | pip install wheel 139 | echo ::endgroup:: 140 | 141 | - name: Before Install 142 | run: .github/workflows/scripts/before_install.sh 143 | shell: bash 144 | 145 | - name: Install 146 | run: .github/workflows/scripts/install.sh 147 | env: 148 | PY_COLORS: '1' 149 | ANSIBLE_FORCE_COLOR: '1' 150 | shell: bash 151 | 152 | - name: Install Python client 153 | run: .github/workflows/scripts/install_python_client.sh 154 | 155 | - name: Install Ruby client 156 | if: ${{ env.TEST == 'bindings' }} || env.TEST == 'generate-bindings' }} 157 | run: .github/workflows/scripts/install_ruby_client.sh 158 | 159 | - name: Before Script 160 | run: .github/workflows/scripts/before_script.sh 161 | 162 | - name: Setting secrets 163 | run: python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" 164 | env: 165 | SECRETS_CONTEXT: ${{ toJson(secrets) }} 166 | 167 | 168 | - name: Download Ruby client 169 | uses: actions/download-artifact@v2 170 | with: 171 | name: python-client.tar 172 | 173 | - name: Untar Ruby client packages 174 | run: tar -xvf ruby-client.tar 175 | 176 | - name: Publish client to rubygems 177 | run: bash .github/workflows/scripts/publish_client_gem.sh 178 | 179 | 180 | 181 | - name: Download Python client 182 | uses: actions/download-artifact@v2 183 | with: 184 | name: python-client.tar 185 | 186 | - name: Untar python client packages 187 | run: tar -xvf python-client.tar 188 | 189 | - name: Publish client to pypi 190 | run: bash .github/workflows/scripts/publish_client_pypi.sh 191 | 192 | 193 | 194 | - name: Download built docs 195 | uses: actions/download-artifact@v2 196 | with: 197 | name: docs.tar 198 | 199 | - name: Publish docs to pulpproject.org 200 | run: | 201 | tar -xvf docs.tar -C ./docs 202 | .github/workflows/scripts/publish_docs.sh nightly ${GITHUB_REF##*/} 203 | 204 | -------------------------------------------------------------------------------- /.github/workflows/scripts/release.py: -------------------------------------------------------------------------------- 1 | # WARNING: DO NOT EDIT! 2 | # 3 | # This file was generated by plugin_template, and is managed by it. Please use 4 | # './plugin-template --github pulp_python' to update this file. 5 | # 6 | # For more info visit https://github.com/pulp/plugin_template 7 | 8 | import argparse 9 | import asyncio 10 | import re 11 | import os 12 | import shutil 13 | import textwrap 14 | 15 | from bandersnatch.mirror import BandersnatchMirror 16 | from bandersnatch.master import Master 17 | from bandersnatch.configuration import BandersnatchConfig 18 | 19 | from git import Repo 20 | 21 | from packaging.requirements import Requirement 22 | 23 | 24 | async def get_package_from_pypi(package_name, plugin_path): 25 | """ 26 | Download a package from PyPI. 27 | 28 | :param name: name of the package to download from PyPI 29 | :return: String path to the package 30 | """ 31 | config = BandersnatchConfig().config 32 | config["mirror"]["master"] = "https://pypi.org" 33 | config["mirror"]["workers"] = "1" 34 | config["mirror"]["directory"] = plugin_path 35 | if not config.has_section("plugins"): 36 | config.add_section("plugins") 37 | config["plugins"]["enabled"] = "blocklist_release\n" 38 | if not config.has_section("allowlist"): 39 | config.add_section("allowlist") 40 | config["plugins"]["enabled"] += "allowlist_release\nallowlist_project\n" 41 | config["allowlist"]["packages"] = "\n".join([package_name]) 42 | os.makedirs(os.path.join(plugin_path, "dist"), exist_ok=True) 43 | async with Master("https://pypi.org/") as master: 44 | mirror = BandersnatchMirror(homedir=plugin_path, master=master) 45 | name = Requirement(package_name).name 46 | result = await mirror.synchronize([name]) 47 | package_found = False 48 | 49 | for package in result[name]: 50 | current_path = os.path.join(plugin_path, package) 51 | destination_path = os.path.join(plugin_path, "dist", os.path.basename(package)) 52 | shutil.move(current_path, destination_path) 53 | package_found = True 54 | return package_found 55 | 56 | 57 | def create_release_commits(repo, release_version, plugin_path): 58 | """Build changelog, set version, commit, bump to next dev version, commit.""" 59 | # First commit: changelog 60 | os.system(f"towncrier --yes --version {release_version}") 61 | git = repo.git 62 | git.add("CHANGES.rst") 63 | git.add("CHANGES/*") 64 | git.commit("-m", f"Building changelog for {release_version}\n\n[noissue]") 65 | 66 | # Second commit: release version 67 | os.system("bump2version release --allow-dirty") 68 | 69 | git.add(f"{plugin_path}/{plugin_name}/*") 70 | git.add(f"{plugin_path}/docs/conf.py") 71 | git.add(f"{plugin_path}/setup.py") 72 | git.add(f"{plugin_path}/requirements.txt") 73 | git.add(f"{plugin_path}/.bumpversion.cfg") 74 | 75 | git.commit("-m", f"Release {release_version}\n\n[noissue]") 76 | 77 | sha = repo.head.object.hexsha 78 | short_sha = git.rev_parse(sha, short=7) 79 | 80 | os.system("bump2version patch --allow-dirty") 81 | 82 | new_dev_version = None 83 | with open(f"{plugin_path}/setup.py") as fp: 84 | for line in fp.readlines(): 85 | if "version=" in line: 86 | new_dev_version = re.split("\"|'", line)[1] 87 | if not new_dev_version: 88 | raise RuntimeError("Could not detect new dev version ... aborting.") 89 | 90 | git.add(f"{plugin_path}/{plugin_name}/*") 91 | git.add(f"{plugin_path}/docs/conf.py") 92 | git.add(f"{plugin_path}/setup.py") 93 | git.add(f"{plugin_path}/requirements.txt") 94 | git.add(f"{plugin_path}/.bumpversion.cfg") 95 | git.commit("-m", f"Bump to {new_dev_version}\n\n[noissue]") 96 | 97 | print(f"Release commit == {short_sha}") 98 | print(f"All changes were committed on branch: release_{release_version}") 99 | return sha 100 | 101 | 102 | def create_tag_and_build_package(repo, desired_tag, commit_sha, plugin_path): 103 | """Create a tag if one is needed and build a package if one is not on PyPI.""" 104 | # Remove auth header config 105 | with repo.config_writer() as conf: 106 | conf.remove_section('http "https://github.com/"') 107 | conf.release() 108 | 109 | # Determine if a tag exists and if it matches the specified commit sha 110 | tag = None 111 | for existing_tag in repo.tags: 112 | if existing_tag.name == desired_tag: 113 | if existing_tag.commit.hexsha == commit_sha: 114 | tag = existing_tag 115 | else: 116 | raise RuntimeError( 117 | "The '{desired_tag}' tag already exists, but the commit sha does not match " 118 | "'{commit_sha}'." 119 | ) 120 | 121 | # Create a tag if one does not exist 122 | if not tag: 123 | tag = repo.create_tag(desired_tag, ref=commit_sha) 124 | 125 | # Checkout the desired tag and reset the tree 126 | repo.head.reference = tag.commit 127 | repo.head.reset(index=True, working_tree=True) 128 | 129 | # Check if Package is available on PyPI 130 | loop = asyncio.get_event_loop() # noqa 131 | # fmt: off 132 | package_found = asyncio.run( 133 | get_package_from_pypi("pulp-python=={tag.name}", plugin_path) 134 | ) # noqa 135 | # fmt: on 136 | if not package_found: 137 | os.system("python3 setup.py sdist bdist_wheel --python-tag py3") 138 | 139 | 140 | helper = textwrap.dedent( 141 | """\ 142 | Start the release process. 143 | 144 | Example: 145 | setup.py on plugin before script: 146 | version="2.0.0.dev" 147 | 148 | $ python .ci/scripts/release.py 149 | 150 | setup.py on plugin after script: 151 | version="2.0.1.dev" 152 | 153 | 154 | """ 155 | ) 156 | parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter, description=helper) 157 | 158 | parser.add_argument( 159 | "release_version", 160 | type=str, 161 | help="The version string for the release.", 162 | ) 163 | 164 | args = parser.parse_args() 165 | 166 | release_version_arg = args.release_version 167 | 168 | release_path = os.path.dirname(os.path.abspath(__file__)) 169 | plugin_path = release_path.split("/.github")[0] 170 | 171 | plugin_name = "pulp_python" 172 | version = None 173 | with open(f"{plugin_path}/setup.py") as fp: 174 | for line in fp.readlines(): 175 | if "version=" in line: 176 | version = re.split("\"|'", line)[1] 177 | if not version: 178 | raise RuntimeError("Could not detect existing version ... aborting.") 179 | release_version = version.replace(".dev", "") 180 | 181 | print(f"\n\nRepo path: {plugin_path}") 182 | repo = Repo(plugin_path) 183 | 184 | release_commit = None 185 | if release_version != release_version_arg: 186 | # Look for a commit with the requested release version 187 | for commit in repo.iter_commits(): 188 | if f"Release {release_version_arg}\n" in commit.message: 189 | release_commit = commit 190 | release_version = release_version_arg 191 | break 192 | if not release_commit: 193 | raise RuntimeError( 194 | f"The release version {release_version_arg} does not match the .dev version at HEAD. " 195 | "A release commit for such version does not exist." 196 | ) 197 | 198 | if not release_commit: 199 | release_commit_sha = create_release_commits(repo, release_version, plugin_path) 200 | else: 201 | release_commit_sha = release_commit.hexsha 202 | create_tag_and_build_package(repo, release_version, release_commit_sha, plugin_path) 203 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Pulp Python Support documentation build configuration file, created by 4 | # sphinx-quickstart on Thu Nov 20 17:53:15 2014. 5 | # 6 | # This file is execfile()d with the current directory set to its containing dir. 7 | # 8 | # Note that not all possible configuration values are present in this 9 | # autogenerated file. 10 | # 11 | # All configuration values have a default; values that are commented out 12 | # serve to show the default. 13 | 14 | import os 15 | import sys 16 | sys.path.insert(0, os.path.abspath('..')) # noqa 17 | 18 | 19 | try: 20 | import sphinx_rtd_theme 21 | except ImportError: 22 | sphinx_rtd_theme = False 23 | 24 | # If extensions (or modules to document with autodoc) are in another directory, 25 | # add these directories to sys.path here. If the directory is relative to the 26 | # documentation root, use os.path.abspath to make it absolute, like shown here. 27 | #sys.path.insert(0, os.path.abspath('.')) 28 | 29 | # -- General configuration ----------------------------------------------------- 30 | 31 | # If your documentation needs a minimal Sphinx version, state it here. 32 | #needs_sphinx = '1.0' 33 | 34 | # Add any Sphinx extension module names here, as strings. They can be extensions 35 | # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. 36 | extensions = ['sphinx.ext.extlinks'] 37 | 38 | # Add any paths that contain templates here, relative to this directory. 39 | templates_path = ['_templates'] 40 | 41 | # The suffix of source filenames. 42 | source_suffix = '.rst' 43 | 44 | # The encoding of source files. 45 | #source_encoding = 'utf-8-sig' 46 | 47 | # The master toctree document. 48 | master_doc = 'index' 49 | 50 | # General information about the project. 51 | project = u'Pulp python Support' 52 | copyright = u'' 53 | 54 | # The version info for the project you're documenting, acts as replacement for 55 | # |version| and |release|, also used in various other places throughout the 56 | # built documents. 57 | # 58 | # The short X.Y version. 59 | version = "3.5.0.dev" 60 | # The full version, including alpha/beta/rc tags. 61 | release = "3.5.0.dev" 62 | 63 | # The language for content autogenerated by Sphinx. Refer to documentation 64 | # for a list of supported languages. 65 | #language = None 66 | 67 | # There are two options for replacing |today|: either, you set today to some 68 | # non-false value, then it is used: 69 | #today = '' 70 | # Else, today_fmt is used as the format for a strftime call. 71 | #today_fmt = '%B %d, %Y' 72 | 73 | # List of patterns, relative to source directory, that match files and 74 | # directories to ignore when looking for source files. 75 | patterns = ['_build'] 76 | 77 | # The reST default role (used for this markup: `text`) to use for all documents. 78 | #default_role = None 79 | 80 | # If true, '()' will be appended to :func: etc. cross-reference text. 81 | #add_function_parentheses = True 82 | 83 | # If true, the current module name will be prepended to all description 84 | # unit titles (such as .. function::). 85 | #add_module_names = True 86 | 87 | # If true, sectionauthor and moduleauthor directives will be shown in the 88 | # output. They are ignored by default. 89 | #show_authors = False 90 | 91 | # The name of the Pygments (syntax highlighting) style to use. 92 | pygments_style = 'sphinx' 93 | 94 | # A list of ignored prefixes for module index sorting. 95 | #modindex_common_prefix = [] 96 | 97 | 98 | # -- Options for HTML output --------------------------------------------------- 99 | 100 | # The theme to use for HTML and HTML Help pages. See the documentation for 101 | # a list of builtin themes. 102 | html_theme = 'sphinx_rtd_theme' if sphinx_rtd_theme else 'default' 103 | 104 | # Theme options are theme-specific and customize the look and feel of a theme 105 | # further. For a list of options available for each theme, see the 106 | # documentation. 107 | #html_theme_options = {} 108 | 109 | # Add any paths that contain custom themes here, relative to this directory. 110 | #html_theme_path = [] 111 | 112 | # The name for this set of Sphinx documents. If None, it defaults to 113 | # " v documentation". 114 | #html_title = None 115 | 116 | # A shorter title for the navigation bar. Default is the same as html_title. 117 | #html_short_title = None 118 | 119 | # The name of an image file (relative to this directory) to place at the top 120 | # of the sidebar. 121 | #html_logo = None 122 | 123 | # The name of an image file (within the static path) to use as favicon of the 124 | # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 125 | # pixels large. 126 | #html_favicon = None 127 | 128 | # Add any paths that contain custom static files (such as style sheets) here, 129 | # relative to this directory. They are copied after the builtin static files, 130 | # so a file named "default.css" will overwrite the builtin "default.css". 131 | html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] if sphinx_rtd_theme else [] 132 | 133 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, 134 | # using the given strftime format. 135 | #html_last_updated_fmt = '%b %d, %Y' 136 | 137 | # If true, SmartyPants will be used to convert quotes and dashes to 138 | # typographically correct entities. 139 | #html_use_smartypants = True 140 | 141 | # Custom sidebar templates, maps document names to template names. 142 | #html_sidebars = {} 143 | 144 | # Additional templates that should be rendered to pages, maps page names to 145 | # template names. 146 | html_additional_pages = {'restapi': 'restapi.html'} 147 | 148 | html_static_path = ['_static'] 149 | 150 | html_js_files = ['survey_banner.js'] 151 | 152 | # If false, no module index is generated. 153 | #html_domain_indices = True 154 | 155 | # If false, no index is generated. 156 | #html_use_index = True 157 | 158 | # If true, the index is split into individual pages for each letter. 159 | #html_split_index = False 160 | 161 | # If true, links to the reST sources are added to the pages. 162 | #html_show_sourcelink = True 163 | 164 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. 165 | #html_show_sphinx = True 166 | 167 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. 168 | #html_show_copyright = True 169 | 170 | # If true, an OpenSearch description file will be output, and all pages will 171 | # contain a tag referring to it. The value of this option must be the 172 | # base URL from which the finished HTML is served. 173 | #html_use_opensearch = '' 174 | 175 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 176 | #html_file_suffix = None 177 | 178 | # -- Options for LaTeX output -------------------------------------------------- 179 | 180 | latex_elements = { 181 | # The paper size ('letterpaper' or 'a4paper'). 182 | #'papersize': 'letterpaper', 183 | 184 | # The font size ('10pt', '11pt' or '12pt'). 185 | #'pointsize': '10pt', 186 | 187 | # Additional stuff for the LaTeX preamble. 188 | #'preamble': '', 189 | } 190 | 191 | # The name of an image file (relative to this directory) to place at the top of 192 | # the title page. 193 | #latex_logo = None 194 | 195 | # For "manual" documents, if this is true, then toplevel headings are parts, 196 | # not chapters. 197 | #latex_use_parts = False 198 | 199 | # If true, show page references after internal links. 200 | #latex_show_pagerefs = False 201 | 202 | # If true, show URL addresses after external links. 203 | #latex_show_urls = False 204 | 205 | # Documents to append as an appendix to all manuals. 206 | #latex_appendices = [] 207 | 208 | # If false, no module index is generated. 209 | #latex_domain_indices = True 210 | 211 | 212 | # If true, show URL addresses after external links. 213 | #man_show_urls = False 214 | 215 | # Documents to append as an appendix to all manuals. 216 | #texinfo_appendices = [] 217 | 218 | # If false, no module index is generated. 219 | #texinfo_domain_indices = True 220 | 221 | # How to display URL addresses: 'footnote', 'no', or 'inline'. 222 | #texinfo_show_urls = 'footnote' 223 | -------------------------------------------------------------------------------- /.github/workflows/scripts/docs-publisher.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # WARNING: DO NOT EDIT! 4 | # 5 | # This file was generated by plugin_template, and is managed by it. Please use 6 | # './plugin-template --github pulp_python' to update this file. 7 | # 8 | # For more info visit https://github.com/pulp/plugin_template 9 | 10 | import argparse 11 | import subprocess 12 | import os 13 | import re 14 | from shutil import rmtree 15 | import tempfile 16 | import requests 17 | import json 18 | from packaging import version 19 | 20 | WORKING_DIR = os.environ["WORKSPACE"] 21 | 22 | VERSION_REGEX = r"(\s*)(version)(\s*)(=)(\s*)(['\"])(.*)(['\"])(.*)" 23 | RELEASE_REGEX = r"(\s*)(release)(\s*)(=)(\s*)(['\"])(.*)(['\"])(.*)" 24 | 25 | USERNAME = "doc_builder_pulp_python" 26 | HOSTNAME = "8.43.85.236" 27 | 28 | SITE_ROOT = "/var/www/docs.pulpproject.org/pulp_python/" 29 | 30 | 31 | def make_directory_with_rsync(remote_paths_list): 32 | """ 33 | Ensure the remote directory path exists. 34 | 35 | :param remote_paths_list: The list of parameters. e.g. ['en', 'latest'] to be en/latest on the 36 | remote. 37 | :type remote_paths_list: a list of strings, with each string representing a directory. 38 | """ 39 | try: 40 | tempdir_path = tempfile.mkdtemp() 41 | cwd = os.getcwd() 42 | os.chdir(tempdir_path) 43 | os.makedirs(os.sep.join(remote_paths_list)) 44 | remote_path_arg = "%s@%s:%s%s" % ( 45 | USERNAME, 46 | HOSTNAME, 47 | SITE_ROOT, 48 | remote_paths_list[0], 49 | ) 50 | local_path_arg = tempdir_path + os.sep + remote_paths_list[0] + os.sep 51 | rsync_command = ["rsync", "-avzh", local_path_arg, remote_path_arg] 52 | exit_code = subprocess.call(rsync_command) 53 | if exit_code != 0: 54 | raise RuntimeError("An error occurred while creating remote directories.") 55 | finally: 56 | rmtree(tempdir_path) 57 | os.chdir(cwd) 58 | 59 | 60 | def ensure_dir(target_dir, clean=True): 61 | """ 62 | Ensure that the directory specified exists and is empty. 63 | 64 | By default this will delete the directory if it already exists 65 | 66 | :param target_dir: The directory to process 67 | :type target_dir: str 68 | :param clean: Whether or not the directory should be removed and recreated 69 | :type clean: bool 70 | """ 71 | if clean: 72 | rmtree(target_dir, ignore_errors=True) 73 | try: 74 | os.makedirs(target_dir) 75 | except OSError: 76 | pass 77 | 78 | 79 | def main(): 80 | """ 81 | Builds documentation using the 'make html' command and rsyncs to docs.pulpproject.org. 82 | """ 83 | parser = argparse.ArgumentParser() 84 | parser.add_argument("--build-type", required=True, help="Build type: nightly or beta.") 85 | parser.add_argument("--branch", required=True, help="Branch or tag name.") 86 | opts = parser.parse_args() 87 | if opts.build_type not in ["nightly", "tag"]: 88 | raise RuntimeError("Build type must be either 'nightly' or 'tag'.") 89 | 90 | build_type = opts.build_type 91 | 92 | branch = opts.branch 93 | 94 | ga_build = False 95 | 96 | publish_at_root = False 97 | 98 | if (not re.search("[a-zA-Z]", branch) or "post" in branch) and len(branch.split(".")) > 2: 99 | ga_build = True 100 | # Only publish docs at the root if this is the latest version 101 | r = requests.get("https://pypi.org/pypi/pulp-python/json") 102 | latest_version = version.parse(json.loads(r.text)["info"]["version"]) 103 | docs_version = version.parse(branch) 104 | # This is to mitigate delays on PyPI which doesn't update metadata in timely manner. 105 | # It doesn't prevent incorrect docs being published at root if 2 releases are done close 106 | # to each other, within PyPI delay. E.g. Release 3.11.0 an then 3.10.1 immediately after. 107 | if docs_version >= latest_version: 108 | publish_at_root = True 109 | # Post releases should use the x.y.z part of the version string to form a path 110 | if "post" in branch: 111 | branch = ".".join(branch.split(".")[:-1]) 112 | 113 | # rsync the docs 114 | print("rsync the docs") 115 | docs_directory = os.sep.join([WORKING_DIR, "docs"]) 116 | local_path_arg = os.sep.join([docs_directory, "_build", "html"]) + os.sep 117 | if build_type != "tag": 118 | # This is a nightly build 119 | remote_path_arg = "%s@%s:%sen/%s/%s/" % ( 120 | USERNAME, 121 | HOSTNAME, 122 | SITE_ROOT, 123 | branch, 124 | build_type, 125 | ) 126 | make_directory_with_rsync(["en", branch, build_type]) 127 | rsync_command = ["rsync", "-avzh", "--delete", local_path_arg, remote_path_arg] 128 | exit_code = subprocess.call(rsync_command, cwd=docs_directory) 129 | if exit_code != 0: 130 | raise RuntimeError("An error occurred while pushing docs.") 131 | elif ga_build: 132 | # This is a GA build. 133 | # publish to the root of docs.pulpproject.org 134 | if publish_at_root: 135 | version_components = branch.split(".") 136 | x_y_version = "{}.{}".format(version_components[0], version_components[1]) 137 | remote_path_arg = "%s@%s:%s" % (USERNAME, HOSTNAME, SITE_ROOT) 138 | rsync_command = [ 139 | "rsync", 140 | "-avzh", 141 | "--delete", 142 | "--exclude", 143 | "en", 144 | "--omit-dir-times", 145 | local_path_arg, 146 | remote_path_arg, 147 | ] 148 | exit_code = subprocess.call(rsync_command, cwd=docs_directory) 149 | if exit_code != 0: 150 | raise RuntimeError("An error occurred while pushing docs.") 151 | # publish to docs.pulpproject.org/en/3.y/ 152 | make_directory_with_rsync(["en", x_y_version]) 153 | remote_path_arg = "%s@%s:%sen/%s/" % ( 154 | USERNAME, 155 | HOSTNAME, 156 | SITE_ROOT, 157 | x_y_version, 158 | ) 159 | rsync_command = [ 160 | "rsync", 161 | "-avzh", 162 | "--delete", 163 | "--omit-dir-times", 164 | local_path_arg, 165 | remote_path_arg, 166 | ] 167 | exit_code = subprocess.call(rsync_command, cwd=docs_directory) 168 | if exit_code != 0: 169 | raise RuntimeError("An error occurred while pushing docs.") 170 | # publish to docs.pulpproject.org/en/3.y.z/ 171 | make_directory_with_rsync(["en", branch]) 172 | remote_path_arg = "%s@%s:%sen/%s/" % (USERNAME, HOSTNAME, SITE_ROOT, branch) 173 | rsync_command = [ 174 | "rsync", 175 | "-avzh", 176 | "--delete", 177 | "--omit-dir-times", 178 | local_path_arg, 179 | remote_path_arg, 180 | ] 181 | exit_code = subprocess.call(rsync_command, cwd=docs_directory) 182 | if exit_code != 0: 183 | raise RuntimeError("An error occurred while pushing docs.") 184 | else: 185 | # This is a pre-release 186 | make_directory_with_rsync(["en", branch]) 187 | remote_path_arg = "%s@%s:%sen/%s/%s/" % ( 188 | USERNAME, 189 | HOSTNAME, 190 | SITE_ROOT, 191 | branch, 192 | build_type, 193 | ) 194 | rsync_command = [ 195 | "rsync", 196 | "-avzh", 197 | "--delete", 198 | "--exclude", 199 | "nightly", 200 | "--exclude", 201 | "testing", 202 | local_path_arg, 203 | remote_path_arg, 204 | ] 205 | exit_code = subprocess.call(rsync_command, cwd=docs_directory) 206 | if exit_code != 0: 207 | raise RuntimeError("An error occurred while pushing docs.") 208 | 209 | 210 | if __name__ == "__main__": 211 | main() 212 | -------------------------------------------------------------------------------- /pulp_python/app/models.py: -------------------------------------------------------------------------------- 1 | import json 2 | from logging import getLogger 3 | 4 | from google.cloud import pubsub_v1 5 | 6 | from aiohttp.web import json_response 7 | from dynaconf import settings 8 | from django.contrib.postgres.fields import ArrayField, JSONField 9 | from django.db import models 10 | 11 | from pulpcore.plugin.models import ( 12 | Content, 13 | Publication, 14 | Distribution, 15 | Remote, 16 | Repository 17 | ) 18 | 19 | from pathlib import PurePath 20 | from .utils import python_content_to_json, PYPI_LAST_SERIAL, PYPI_SERIAL_CONSTANT 21 | from pulpcore.plugin.repo_version_utils import remove_duplicates, validate_repo_version 22 | 23 | log = getLogger(__name__) 24 | 25 | 26 | PACKAGE_TYPES = ( 27 | ("bdist_dmg", "bdist_dmg"), 28 | ("bdist_dumb", "bdist_dumb"), 29 | ("bdist_egg", "bdist_egg"), 30 | ("bdist_msi", "bdist_msi"), 31 | ("bdist_rpm", "bdist_rpm"), 32 | ("bdist_wheel", "bdist_wheel"), 33 | ("bdist_wininst", "bdist_wininst"), 34 | ("sdist", "sdist"), 35 | ) 36 | 37 | PLATFORMS = (("windows", "windows"), 38 | ("macos", "macos"), 39 | ("freebsd", "freebsd"), 40 | ("linux", "linux")) 41 | 42 | 43 | class PythonDistribution(Distribution): 44 | """ 45 | Distribution for 'Python' Content. 46 | """ 47 | 48 | TYPE = 'python' 49 | 50 | allow_uploads = models.BooleanField(default=True) 51 | 52 | def content_handler(self, path): 53 | """ 54 | Handler to serve extra, non-Artifact content for this Distribution 55 | 56 | Args: 57 | path (str): The path being requested 58 | Returns: 59 | None if there is no content to be served at path. Otherwise a 60 | aiohttp.web_response.Response with the content. 61 | """ 62 | path = PurePath(path) 63 | name = None 64 | version = None 65 | if path.match("pypi/*/*/json"): 66 | version = path.parts[2] 67 | name = path.parts[1] 68 | elif path.match("pypi/*/json"): 69 | name = path.parts[1] 70 | if path.match("*.tar.gz") or path.match("*.whl"): 71 | try: 72 | project_id = settings.GOOGLE_PUBSUB_PROJECT_ID 73 | topic_id = settings.GOOGLE_PUBSUB_TOPIC_ID 74 | 75 | publisher = pubsub_v1.PublisherClient() 76 | topic_path = publisher.topic_path(project_id, topic_id) 77 | 78 | message = json.dumps({ 79 | "action": "package_requested", 80 | "language": "python", 81 | "package": str(path), 82 | "source": self.base_path, 83 | }) 84 | 85 | response = publisher.publish(topic_path, message.encode("utf-8")) 86 | log.info( 87 | "package_requested message send to %s pub/sub, %s", 88 | topic_id, 89 | response.result() 90 | ) 91 | except Exception: 92 | log.exception( 93 | "Could not call package_requested message to pub/sub server" 94 | ) 95 | if name: 96 | package_content = PythonPackageContent.objects.filter( 97 | pk__in=self.publication.repository_version.content, 98 | name__iexact=name 99 | ) 100 | # TODO Change this value to the Repo's serial value when implemented 101 | headers = {PYPI_LAST_SERIAL: str(PYPI_SERIAL_CONSTANT)} 102 | json_body = python_content_to_json(self.base_path, package_content, version=version) 103 | if json_body: 104 | return json_response(json_body, headers=headers) 105 | 106 | return None 107 | 108 | class Meta: 109 | default_related_name = "%(app_label)s_%(model_name)s" 110 | 111 | 112 | class PythonPackageContent(Content): 113 | """ 114 | A Content Type representing Python's Distribution Package. 115 | 116 | As defined in pep-0426 and pep-0345. 117 | 118 | https://www.python.org/dev/peps/pep-0491/ 119 | https://www.python.org/dev/peps/pep-0345/ 120 | """ 121 | 122 | TYPE = 'python' 123 | repo_key_fields = ("filename",) 124 | # Required metadata 125 | filename = models.TextField(db_index=True) 126 | packagetype = models.TextField(choices=PACKAGE_TYPES) 127 | name = models.TextField() 128 | version = models.TextField() 129 | sha256 = models.CharField(unique=True, db_index=True, max_length=64) 130 | # Optional metadata 131 | python_version = models.TextField() 132 | metadata_version = models.TextField() 133 | summary = models.TextField() 134 | description = models.TextField() 135 | keywords = models.TextField() 136 | home_page = models.TextField() 137 | download_url = models.TextField() 138 | author = models.TextField() 139 | author_email = models.TextField() 140 | maintainer = models.TextField() 141 | maintainer_email = models.TextField() 142 | license = models.TextField() 143 | requires_python = models.TextField() 144 | project_url = models.TextField() 145 | platform = models.TextField() 146 | supported_platform = models.TextField() 147 | requires_dist = JSONField(default=list) 148 | provides_dist = JSONField(default=list) 149 | obsoletes_dist = JSONField(default=list) 150 | requires_external = JSONField(default=list) 151 | classifiers = JSONField(default=list) 152 | project_urls = JSONField(default=dict) 153 | description_content_type = models.TextField() 154 | 155 | def __str__(self): 156 | """ 157 | Provide more useful repr information. 158 | 159 | Overrides Content.str to provide the distribution version and type at 160 | the end. 161 | 162 | e.g. 163 | 164 | """ 165 | return '<{obj_name}: {name} [{version}] ({type})>'.format( 166 | obj_name=self._meta.object_name, 167 | name=self.name, 168 | version=self.version, 169 | type=self.packagetype 170 | ) 171 | 172 | class Meta: 173 | default_related_name = "%(app_label)s_%(model_name)s" 174 | unique_together = ("sha256",) 175 | 176 | 177 | class PythonPublication(Publication): 178 | """ 179 | A Publication for PythonContent. 180 | """ 181 | 182 | TYPE = 'python' 183 | 184 | class Meta: 185 | default_related_name = "%(app_label)s_%(model_name)s" 186 | 187 | 188 | class PythonRemote(Remote): 189 | """ 190 | A Remote for Python Content. 191 | 192 | Fields: 193 | 194 | prereleases (models.BooleanField): Whether to sync pre-release versions of packages. 195 | """ 196 | 197 | TYPE = 'python' 198 | DEFAULT_DOWNLOAD_CONCURRENCY = 10 199 | prereleases = models.BooleanField(default=False) 200 | includes = JSONField(default=list) 201 | excludes = JSONField(default=list) 202 | package_types = ArrayField(models.CharField(max_length=15, blank=True), 203 | choices=PACKAGE_TYPES, default=list) 204 | keep_latest_packages = models.IntegerField(default=0) 205 | exclude_platforms = ArrayField(models.CharField(max_length=10, blank=True), 206 | choices=PLATFORMS, default=list) 207 | 208 | class Meta: 209 | default_related_name = "%(app_label)s_%(model_name)s" 210 | 211 | 212 | class PythonRepository(Repository): 213 | """ 214 | Repository for "python" content. 215 | """ 216 | 217 | TYPE = "python" 218 | CONTENT_TYPES = [PythonPackageContent] 219 | REMOTE_TYPES = [PythonRemote] 220 | 221 | autopublish = models.BooleanField(default=False) 222 | 223 | class Meta: 224 | default_related_name = "%(app_label)s_%(model_name)s" 225 | 226 | def on_new_version(self, version): 227 | """ 228 | Called when new repository versions are created. 229 | 230 | Args: 231 | version: The new repository version 232 | """ 233 | super().on_new_version(version) 234 | 235 | # avoid circular import issues 236 | from pulp_python.app import tasks 237 | 238 | if self.autopublish: 239 | tasks.publish(repository_version_pk=version.pk) 240 | 241 | def finalize_new_version(self, new_version): 242 | """ 243 | Remove duplicate packages that have the same filename. 244 | """ 245 | remove_duplicates(new_version) 246 | validate_repo_version(new_version) 247 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | # WARNING: DO NOT EDIT! 2 | # 3 | # This file was generated by plugin_template, and is managed by it. Please use 4 | # './plugin-template --github pulp_python' to update this file. 5 | # 6 | # For more info visit https://github.com/pulp/plugin_template 7 | --- 8 | name: Release Pipeline 9 | on: 10 | workflow_dispatch: 11 | inputs: 12 | release: 13 | description: "Release tag (e.g. 3.2.1)" 14 | required: true 15 | 16 | env: 17 | RELEASE_WORKFLOW: true 18 | 19 | jobs: 20 | build-artifacts: 21 | runs-on: ubuntu-latest 22 | 23 | strategy: 24 | fail-fast: false 25 | 26 | steps: 27 | - uses: actions/checkout@v2 28 | with: 29 | # by default, it uses a depth of 1 30 | # this fetches all history so that we can read each commit 31 | fetch-depth: 0 32 | 33 | - uses: actions/setup-python@v2 34 | with: 35 | python-version: "3.7" 36 | 37 | - name: Install python dependencies 38 | run: | 39 | echo ::group::PYDEPS 40 | pip install bandersnatch bump2version gitpython python-redmine towncrier wheel 41 | echo ::endgroup:: 42 | 43 | - name: Configure Git with pulpbot name and email 44 | run: | 45 | git config --global user.name 'pulpbot' 46 | git config --global user.email 'pulp-infra@redhat.com' 47 | 48 | - name: Setting secrets 49 | run: python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" 50 | env: 51 | SECRETS_CONTEXT: ${{ toJson(secrets) }} 52 | 53 | - name: Create the release commit, tag it, create a post-release commit, and build plugin package 54 | run: python .github/workflows/scripts/release.py ${{ github.event.inputs.release }} 55 | 56 | - name: 'Tar files' 57 | run: tar -cvf pulp_python.tar $GITHUB_WORKSPACE 58 | 59 | - name: 'Upload Artifact' 60 | uses: actions/upload-artifact@v2 61 | with: 62 | name: pulp_python.tar 63 | path: pulp_python.tar 64 | test: 65 | needs: build-artifacts 66 | 67 | runs-on: ubuntu-latest 68 | 69 | strategy: 70 | fail-fast: false 71 | matrix: 72 | env: 73 | - TEST: pulp 74 | - TEST: docs 75 | - TEST: s3 76 | - TEST: generate-bindings 77 | 78 | steps: 79 | - uses: actions/download-artifact@v2 80 | with: 81 | name: pulp_python.tar 82 | 83 | - uses: actions/setup-python@v2 84 | with: 85 | python-version: "3.7" 86 | - uses: actions/setup-ruby@v1 87 | with: 88 | ruby-version: "2.6" 89 | 90 | - name: Untar repository 91 | run: | 92 | shopt -s dotglob 93 | tar -xf pulp_python.tar 94 | mv home/runner/work/pulp_python/pulp_python/* ./ 95 | 96 | # update to the branch's latest ci files rather than the ones from the release tag. this is 97 | # helpful when there was a problem with the ci files during the release which needs to be 98 | # fixed after the release tag has been created 99 | - name: Update ci files 100 | run: | 101 | git checkout "origin/${GITHUB_REF##*/}" -- .ci 102 | git checkout "origin/${GITHUB_REF##*/}" -- .github 103 | 104 | - name: Install httpie 105 | run: | 106 | echo ::group::HTTPIE 107 | sudo apt-get update -yq 108 | sudo -E apt-get -yq --no-install-suggests --no-install-recommends install httpie 109 | echo ::endgroup:: 110 | echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/.ci/assets/httpie/" >> $GITHUB_ENV 111 | echo "TEST=${{ matrix.env.TEST }}" >> $GITHUB_ENV 112 | 113 | - name: Before Install 114 | run: .github/workflows/scripts/before_install.sh 115 | shell: bash 116 | 117 | - name: Install 118 | run: | 119 | export PLUGIN_VERSION=${{ github.event.inputs.release }} 120 | .github/workflows/scripts/install.sh 121 | env: 122 | PY_COLORS: '1' 123 | ANSIBLE_FORCE_COLOR: '1' 124 | shell: bash 125 | 126 | - name: Before Script 127 | run: | 128 | .github/workflows/scripts/before_script.sh 129 | 130 | - name: Setting secrets 131 | run: python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" 132 | env: 133 | SECRETS_CONTEXT: ${{ toJson(secrets) }} 134 | 135 | - name: Install Python client 136 | run: .github/workflows/scripts/install_python_client.sh 137 | 138 | - name: Install Ruby client 139 | if: ${{ env.TEST == 'bindings' || env.TEST == 'generate-bindings' }} 140 | run: .github/workflows/scripts/install_ruby_client.sh 141 | 142 | - name: Script 143 | if: ${{ env.TEST != 'generate-bindings' }} 144 | run: .github/workflows/scripts/script.sh 145 | shell: bash 146 | 147 | - name: Upload python client packages 148 | if: ${{ env.TEST == 'bindings' || env.TEST == 'generate-bindings' }} 149 | uses: actions/upload-artifact@v2 150 | with: 151 | name: python-client.tar 152 | path: python-client.tar 153 | 154 | - name: Upload ruby client packages 155 | if: ${{ env.TEST == 'bindings' || env.TEST == 'generate-bindings' }} 156 | uses: actions/upload-artifact@v2 157 | with: 158 | name: ruby-client.tar 159 | path: ruby-client.tar 160 | 161 | - name: Upload built docs 162 | if: ${{ env.TEST == 'docs' }} 163 | uses: actions/upload-artifact@v2 164 | with: 165 | name: docs.tar 166 | path: docs/docs.tar 167 | 168 | - name: After failure 169 | if: failure() 170 | run: | 171 | http --timeout 30 --check-status --pretty format --print hb http://pulp/pulp/api/v3/status/ || true 172 | docker images || true 173 | docker ps -a || true 174 | docker logs pulp || true 175 | docker exec pulp ls -latr /etc/yum.repos.d/ || true 176 | docker exec pulp cat /etc/yum.repos.d/* || true 177 | docker exec pulp pip3 list 178 | 179 | 180 | publish: 181 | runs-on: ubuntu-latest 182 | needs: test 183 | 184 | env: 185 | TEST: publish 186 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 187 | 188 | steps: 189 | - uses: actions/download-artifact@v2 190 | with: 191 | name: pulp_python.tar 192 | 193 | - uses: actions/setup-python@v2 194 | with: 195 | python-version: "3.7" 196 | 197 | - uses: actions/setup-ruby@v1 198 | with: 199 | ruby-version: "2.6" 200 | 201 | - name: Untar repository 202 | run: | 203 | shopt -s dotglob 204 | tar -xf pulp_python.tar 205 | mv home/runner/work/pulp_python/pulp_python/* ./ 206 | 207 | # update to the branch's latest ci files rather than the ones from the release tag. this is 208 | # helpful when there was a problem with the ci files during the release which needs to be 209 | # fixed after the release tag has been created 210 | - name: Update ci files 211 | run: | 212 | git checkout "origin/${GITHUB_REF##*/}" -- .ci 213 | git checkout "origin/${GITHUB_REF##*/}" -- .github 214 | 215 | - name: Setting secrets 216 | run: python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" 217 | env: 218 | SECRETS_CONTEXT: ${{ toJson(secrets) }} 219 | 220 | - name: Push branch and tag to GitHub 221 | run: bash .github/workflows/scripts/push_branch_and_tag_to_github.sh ${{ github.event.inputs.release }} 222 | - name: Download built docs 223 | uses: actions/download-artifact@v2 224 | with: 225 | name: docs.tar 226 | 227 | - name: Publish docs to pulpproject.org 228 | run: | 229 | tar -xvf docs.tar -C ./docs 230 | .github/workflows/scripts/publish_docs.sh tag ${{ github.event.inputs.release }} 231 | - name: Deploy plugin to pypi 232 | run: bash .github/workflows/scripts/publish_plugin_pypi.sh ${{ github.event.inputs.release }} 233 | - name: Download Python client 234 | uses: actions/download-artifact@v2 235 | with: 236 | name: python-client.tar 237 | 238 | - name: Untar python client packages 239 | run: tar -xvf python-client.tar 240 | 241 | - name: Publish client to pypi 242 | run: bash .github/workflows/scripts/publish_client_pypi.sh 243 | - name: Download Ruby client 244 | uses: actions/download-artifact@v2 245 | with: 246 | name: ruby-client.tar 247 | 248 | - name: Untar Ruby client packages 249 | run: tar -xvf ruby-client.tar 250 | 251 | - name: Publish client to rubygems 252 | run: bash .github/workflows/scripts/publish_client_gem.sh 253 | 254 | - name: Create release on GitHub 255 | run: bash .github/workflows/scripts/create_release_from_tag.sh ${{ github.event.inputs.release }} 256 | -------------------------------------------------------------------------------- /CHANGES.rst: -------------------------------------------------------------------------------- 1 | ========= 2 | Changelog 3 | ========= 4 | 5 | .. 6 | You should *NOT* be adding new change log entries to this file, this 7 | file is managed by towncrier. You *may* edit previous change logs to 8 | fix problems like typo corrections or such. 9 | To add a new change log entry, please see 10 | https://docs.pulpproject.org/en/3.0/nightly/contributing/git.html#changelog-update 11 | 12 | WARNING: Don't drop the next directive! 13 | 14 | .. towncrier release notes start 15 | 16 | 3.4.0 (2021-06-17) 17 | 18 | Features 19 | -------- 20 | 21 | - Added ``twine`` (and other similar Python tools) package upload support 22 | `#342 `_ 23 | - PyPI endpoints are now available at ``/pypi/{base_path}/`` 24 | `#376 `_ 25 | - Changed the global uniqueness constraint for ``PythonPackageContent`` to its sha256 digest 26 | `#380 `_ 27 | 28 | 29 | Bugfixes 30 | -------- 31 | 32 | - Added missing fields to PyPI live JSON API to be compliant with core metadata version 2.1 33 | `#352 `_ 34 | - Fixed sync to use default concurrency (10) when download_concurrency was not specified 35 | `#391 `_ 36 | 37 | 38 | ---- 39 | 40 | 41 | 3.3.0 (2021-05-27) 42 | ================== 43 | 44 | 45 | Features 46 | -------- 47 | 48 | - Add support for automatic publishing and distributing. 49 | `#365 `_ 50 | 51 | 52 | Bugfixes 53 | -------- 54 | 55 | - Fixed publications publishing more content than was in the repository 56 | `#362 `_ 57 | 58 | 59 | Improved Documentation 60 | ---------------------- 61 | 62 | - Update syntax in doc for cli repository content add command 63 | `#368 `_ 64 | 65 | 66 | Misc 67 | ---- 68 | 69 | - `#347 `_, `#360 `_, `#371 `_ 70 | 71 | 72 | ---- 73 | 74 | 75 | 3.2.0 (2021-04-14) 76 | ================== 77 | 78 | 79 | Features 80 | -------- 81 | 82 | - Added new sync filter `keep_latest_packages` to specify how many latest versions of packages to sync 83 | `#339 `_ 84 | - Added new sync filters `package_types` and `exclude_platforms` to specify package types to sync 85 | `#341 `_ 86 | 87 | 88 | Misc 89 | ---- 90 | 91 | - `#354 `_ 92 | 93 | 94 | ---- 95 | 96 | 97 | 3.1.0 (2021-03-12) 98 | ================== 99 | 100 | 101 | Features 102 | -------- 103 | 104 | - Python content can now be filtered by requires_python 105 | `#3629 `_ 106 | 107 | 108 | Improved Documentation 109 | ---------------------- 110 | 111 | - Updated workflows to use Pulp CLI commands 112 | `#8364 `_ 113 | 114 | 115 | ---- 116 | 117 | 118 | 3.0.0 (2021-01-12) 119 | ================== 120 | 121 | 122 | Bugfixes 123 | -------- 124 | 125 | - Remote proxy settings are now passed to Bandersnatch while syncing 126 | `#7864 `_ 127 | 128 | 129 | Improved Documentation 130 | ---------------------- 131 | 132 | - Added bullet list of Python Plugin features and a tech preview page for new experimental features 133 | `#7628 `_ 134 | 135 | 136 | ---- 137 | 138 | 139 | 3.0.0b12 (2020-11-05) 140 | ===================== 141 | 142 | 143 | Features 144 | -------- 145 | 146 | - Pulp Python can now fully mirror all packages from PyPi 147 | `#985 `_ 148 | - Implemented PyPi's json API at content endpoint '/pypi/{package-name}/json'. Pulp can now perform basic syncing on other Pulp Python instances. 149 | `#2886 `_ 150 | - Pulp Python now uses Bandersnatch to perform syncing and filtering of package metadata 151 | `#6930 `_ 152 | 153 | 154 | Bugfixes 155 | -------- 156 | 157 | - Sync now includes python package's classifiers in the content unit 158 | `#3627 `_ 159 | - Policy can now be specified when creating a remote from a Bandersnatch config 160 | `#7331 `_ 161 | - Includes/excludes/prereleases fields are now properly set in a remote from Bandersnatch config 162 | `#7392 `_ 163 | 164 | 165 | Improved Documentation 166 | ---------------------- 167 | 168 | - Fixed makemigrations commands in the install docs 169 | `#5386 `_ 170 | 171 | 172 | Misc 173 | ---- 174 | 175 | - `#6875 `_, `#7401 `_ 176 | 177 | 178 | ---- 179 | 180 | 181 | 3.0.0b11 (2020-08-18) 182 | ===================== 183 | 184 | 185 | Compatibility update for pulpcore 3.6 186 | 187 | 188 | ---- 189 | 190 | 191 | 3.0.0b10 (2020-08-05) 192 | ===================== 193 | 194 | 195 | Features 196 | -------- 197 | 198 | - Added a new endpoint to remotes "/from_bandersnatch" that allows for Python remote creation from a Bandersnatch config file. 199 | `#6929 `_ 200 | 201 | 202 | Bugfixes 203 | -------- 204 | 205 | - Including requirements.txt on MANIFEST.in 206 | `#6891 `_ 207 | - Updating API to not return publications that aren't complete. 208 | `#6987 `_ 209 | - Fixed an issue that prevented 'on_demand' content from being published. 210 | `#7128 `_ 211 | 212 | 213 | Improved Documentation 214 | ---------------------- 215 | 216 | - Change the commands for publication and distribution on the publish workflow to use their respective scripts already defined in _scripts. 217 | `#6877 `_ 218 | - Updated sync.sh, publication.sh and distribution.sh in docs/_scripts to reference wait_until_task_finished function from base.sh 219 | `#6918 `_ 220 | 221 | 222 | ---- 223 | 224 | 225 | 3.0.0b9 (2020-06-01) 226 | ==================== 227 | 228 | 229 | Features 230 | -------- 231 | 232 | - Add upload functionality to the python contents endpoints. 233 | `#5464 `_ 234 | 235 | 236 | Bugfixes 237 | -------- 238 | 239 | - Fixed the 500 error returned by the OpenAPI schema endpoint. 240 | `#5452 `_ 241 | 242 | 243 | Improved Documentation 244 | ---------------------- 245 | 246 | - Change the prefix of Pulp services from pulp-* to pulpcore-* 247 | `#4554 `_ 248 | - Added "python/python/" to fix two commands in repo.sh, fixed export command in sync.sh 249 | `#6790 `_ 250 | - Added "index.html" to the relative_path field for both project_metadata and index_metadata. Added a "/" to fix the link in the simple_index_template. 251 | `#6792 `_ 252 | - Updated the workflow documentation for upload.html. Fixed the workflow commands and added more details to the instructions. 253 | `#6854 `_ 254 | 255 | 256 | Deprecations and Removals 257 | ------------------------- 258 | 259 | - Change `_id`, `_created`, `_last_updated`, `_href` to `pulp_id`, `pulp_created`, `pulp_last_updated`, `pulp_href` 260 | `#5457 `_ 261 | - Remove "_" from `_versions_href`, `_latest_version_href` 262 | `#5548 `_ 263 | - Removing base field: `_type` . 264 | `#5550 `_ 265 | - Sync is no longer available at the {remote_href}/sync/ repository={repo_href} endpoint. Instead, use POST {repo_href}/sync/ remote={remote_href}. 266 | 267 | Creating / listing / editing / deleting python repositories is now performed on /pulp/api/v3/python/python/ instead of /pulp/api/v3/repositories/. Only python content can be present in a python repository, and only a python repository can hold python content. 268 | `#5625 `_ 269 | 270 | 271 | Misc 272 | ---- 273 | 274 | - `#remotetests `_, `#4681 `_, `#4682 `_, `#5304 `_, `#5471 `_, `#5580 `_, `#5701 `_ 275 | 276 | 277 | ---- 278 | 279 | 280 | 3.0.0b8 (2019-09-16) 281 | ==================== 282 | 283 | 284 | Misc 285 | ---- 286 | 287 | - `#4681 `_ 288 | 289 | 290 | ---- 291 | 292 | 293 | 3.0.0b7 (2019-08-01) 294 | ==================== 295 | 296 | 297 | Features 298 | -------- 299 | 300 | - Users can upload a file to create content and optionally add to a repo in one step known as 301 | one-shot upload 302 | `#4396 `_ 303 | - Override the Remote's serializer to allow policy='on_demand' and policy='streamed'. 304 | `#4990 `_ 305 | 306 | 307 | Improved Documentation 308 | ---------------------- 309 | 310 | - Switch to using `towncrier `_ for better release notes. 311 | `#4875 `_ 312 | 313 | 314 | ---- 315 | 316 | 317 | -------------------------------------------------------------------------------- /pulp_python/app/viewsets.py: -------------------------------------------------------------------------------- 1 | from bandersnatch.configuration import BandersnatchConfig 2 | from drf_spectacular.utils import extend_schema 3 | from rest_framework import status 4 | from rest_framework.decorators import action 5 | from rest_framework.response import Response 6 | 7 | from pulpcore.plugin import viewsets as core_viewsets 8 | from pulpcore.plugin.actions import ModifyRepositoryActionMixin 9 | from pulpcore.plugin.models import RepositoryVersion 10 | from pulpcore.plugin.serializers import ( 11 | AsyncOperationResponseSerializer, 12 | RepositorySyncURLSerializer, 13 | ) 14 | from pulpcore.plugin.tasking import dispatch 15 | 16 | from pulp_python.app import models as python_models 17 | from pulp_python.app import serializers as python_serializers 18 | from pulp_python.app import tasks 19 | 20 | 21 | class PythonRepositoryViewSet(core_viewsets.RepositoryViewSet, ModifyRepositoryActionMixin): 22 | """ 23 | PythonRepository represents a single Python repository, to which content can be 24 | synced, added, or removed. 25 | """ 26 | 27 | endpoint_name = 'python' 28 | queryset = python_models.PythonRepository.objects.all() 29 | serializer_class = python_serializers.PythonRepositorySerializer 30 | 31 | @extend_schema( 32 | summary="Sync from remote", 33 | responses={202: AsyncOperationResponseSerializer} 34 | ) 35 | @action(detail=True, methods=['post'], serializer_class=RepositorySyncURLSerializer) 36 | def sync(self, request, pk): 37 | """ 38 | 39 | Trigger an asynchronous task to sync python content. The sync task will retrieve Python 40 | content from the specified `Remote` and update the specified `Respository`, creating a 41 | new `RepositoryVersion`. 42 | """ 43 | repository = self.get_object() 44 | serializer = RepositorySyncURLSerializer( 45 | data=request.data, 46 | context={'request': request, "repository_pk": pk} 47 | ) 48 | serializer.is_valid(raise_exception=True) 49 | remote = serializer.validated_data.get('remote', repository.remote) 50 | mirror = serializer.validated_data.get('mirror') 51 | 52 | result = dispatch( 53 | tasks.sync, 54 | [repository, remote], 55 | kwargs={ 56 | 'remote_pk': str(remote.pk), 57 | 'repository_pk': str(repository.pk), 58 | 'mirror': mirror 59 | } 60 | ) 61 | return core_viewsets.OperationPostponedResponse(result, request) 62 | 63 | 64 | class PythonRepositoryVersionViewSet(core_viewsets.RepositoryVersionViewSet): 65 | """ 66 | PythonRepositoryVersion represents a single Python repository version. 67 | """ 68 | 69 | parent_viewset = PythonRepositoryViewSet 70 | 71 | 72 | class PythonDistributionViewSet(core_viewsets.DistributionViewSet): 73 | """ 74 | 75 | Pulp Python Distributions are used to distribute Python content from 76 | Python Repositories or 77 | Python Publications. Pulp Python 78 | Distributions should not be confused with "Python Distribution" as defined by the Python 79 | community. In Pulp usage, Python content is referred to as Python Package Content. 81 | """ 82 | 83 | endpoint_name = 'pypi' 84 | queryset = python_models.PythonDistribution.objects.all() 85 | serializer_class = python_serializers.PythonDistributionSerializer 86 | 87 | 88 | class PythonPackageContentFilter(core_viewsets.ContentFilter): 89 | """ 90 | FilterSet for PythonPackageContent. 91 | """ 92 | 93 | class Meta: 94 | model = python_models.PythonPackageContent 95 | fields = { 96 | 'name': ['exact', 'in'], 97 | 'author': ['exact', 'in'], 98 | 'packagetype': ['exact', 'in'], 99 | 'requires_python': ['exact', 'in', "contains"], 100 | 'filename': ['exact', 'in', 'contains'], 101 | 'keywords': ['in', 'contains'], 102 | } 103 | 104 | 105 | class PythonPackageSingleArtifactContentUploadViewSet( 106 | core_viewsets.SingleArtifactContentUploadViewSet): 107 | """ 108 | 109 | PythonPackageContent represents each individually installable Python package. In the Python 110 | ecosystem, this is called a Python Distribution, sometimes (ambiguously) refered to as a 111 | package. In Pulp Python, we refer to it as PythonPackageContent. Each 112 | PythonPackageContent corresponds to a single filename, for example 113 | `pulpcore-3.0.0rc1-py3-none-any.whl` or `pulpcore-3.0.0rc1.tar.gz`. 114 | 115 | """ 116 | 117 | endpoint_name = 'packages' 118 | queryset = python_models.PythonPackageContent.objects.all() 119 | serializer_class = python_serializers.PythonPackageContentSerializer 120 | minimal_serializer_class = python_serializers.MinimalPythonPackageContentSerializer 121 | filterset_class = PythonPackageContentFilter 122 | 123 | 124 | class PythonRemoteViewSet(core_viewsets.RemoteViewSet): 125 | """ 126 | 127 | Python Remotes are representations of an external repository of Python content, eg. 128 | PyPI. Fields include upstream repository config. Python Remotes are also used to `sync` from 129 | upstream repositories, and contains sync settings. 130 | 131 | """ 132 | 133 | endpoint_name = 'python' 134 | queryset = python_models.PythonRemote.objects.all() 135 | serializer_class = python_serializers.PythonRemoteSerializer 136 | 137 | @extend_schema( 138 | summary="Create from Bandersnatch", 139 | responses={201: python_serializers.PythonRemoteSerializer}, 140 | ) 141 | @action(detail=False, methods=["post"], 142 | serializer_class=python_serializers.PythonBanderRemoteSerializer) 143 | def from_bandersnatch(self, request): 144 | """ 145 | 146 | Takes the fields specified in the Bandersnatch config and creates a Python Remote from it. 147 | """ 148 | serializer = self.get_serializer(data=request.data) 149 | serializer.is_valid(raise_exception=True) 150 | bander_config_file = serializer.validated_data.get("config") 151 | name = serializer.validated_data.get("name") 152 | policy = serializer.validated_data.get("policy") 153 | bander_config = BandersnatchConfig(bander_config_file.file.name).config 154 | data = {"name": name, 155 | "policy": policy, 156 | "url": bander_config.get("mirror", "master"), 157 | "download_concurrency": bander_config.get("mirror", "workers"), 158 | } 159 | enabled = bander_config.get("plugins", "enabled") 160 | enabled_all = "all" in enabled 161 | data["prereleases"] = not (enabled_all or "prerelease_release" in enabled) 162 | # TODO refactor to use a translation object 163 | plugin_filters = { # plugin : (section_name, bander_option, pulp_option) 164 | "allowlist_project": ("allowlist", "packages", "includes"), 165 | "blocklist_project": ("blocklist", "packages", "excludes"), 166 | "regex_release_file_metadata": ( 167 | "regex_release_file_metadata", 168 | "any:release_file.packagetype", 169 | "package_types", 170 | ), 171 | "latest_release": ("latest_release", "keep", "keep_latest_packages"), 172 | "exclude_platform": ("blocklist", "platforms", "exclude_platforms"), 173 | } 174 | for plugin, options in plugin_filters.items(): 175 | if (enabled_all or plugin in enabled) and \ 176 | bander_config.has_option(options[0], options[1]): 177 | data[options[2]] = bander_config.get(options[0], options[1]).split() 178 | remote = python_serializers.PythonRemoteSerializer(data=data, context={"request": request}) 179 | remote.is_valid(raise_exception=True) 180 | remote.save() 181 | headers = self.get_success_headers(remote.data) 182 | return Response(remote.data, status=status.HTTP_201_CREATED, headers=headers) 183 | 184 | 185 | class PythonPublicationViewSet(core_viewsets.PublicationViewSet): 186 | """ 187 | 188 | Python Publications refer to the Python Package content in a repository version, and include 189 | metadata about that content. 190 | 191 | """ 192 | 193 | endpoint_name = 'pypi' 194 | queryset = python_models.PythonPublication.objects.exclude(complete=False) 195 | serializer_class = python_serializers.PythonPublicationSerializer 196 | 197 | @extend_schema( 198 | responses={202: AsyncOperationResponseSerializer} 199 | ) 200 | def create(self, request): 201 | """ 202 | 203 | Dispatches a publish task, which generates metadata that will be used by pip. 204 | """ 205 | serializer = self.get_serializer(data=request.data) 206 | serializer.is_valid(raise_exception=True) 207 | repository_version = serializer.validated_data.get('repository_version') 208 | 209 | # Safe because version OR repository is enforced by serializer. 210 | if not repository_version: 211 | repository = serializer.validated_data.get('repository') 212 | repository_version = RepositoryVersion.latest(repository) 213 | 214 | result = dispatch( 215 | tasks.publish, 216 | [repository_version.repository], 217 | kwargs={ 218 | 'repository_version_pk': str(repository_version.pk) 219 | } 220 | ) 221 | return core_viewsets.OperationPostponedResponse(result, request) 222 | --------------------------------------------------------------------------------