├── .ci ├── assets │ ├── .gitkeep │ ├── release_requirements.txt │ ├── httpie │ │ └── config.json │ └── ci_constraints.txt ├── ansible │ ├── ansible.cfg │ ├── inventory.yaml │ ├── filter │ │ └── repr.py │ ├── smash-config.json │ ├── settings.py.j2 │ ├── build_container.yaml │ └── Containerfile.j2 └── scripts │ ├── check_gettext.sh │ ├── update_github.py │ ├── check_pulpcore_imports.sh │ ├── schema.py │ └── pr_labels.py ├── docs ├── admin │ ├── learn │ │ └── .gitkeep │ ├── index.md │ ├── guides │ │ └── add-signing-services.md │ └── reference │ │ └── settings.md ├── user │ ├── reference │ │ └── .gitkeep │ ├── guides │ │ ├── _SUMMARY.md │ │ ├── sign-packages.md │ │ └── alternate-content-source.md │ └── index.md └── dev │ ├── guides │ └── bindings.md │ └── index.md ├── pulp_rpm ├── tests │ ├── __init__.py │ ├── unit │ │ ├── __init__.py │ │ └── test_models.py │ ├── performance │ │ ├── __init__.py │ │ └── test_pulp_to_pulp.py │ ├── functional │ │ ├── content_handler │ │ │ └── __init__.py │ │ ├── __init__.py │ │ ├── api │ │ │ ├── __init__.py │ │ │ ├── test_prn.py │ │ │ ├── test_advisory_conflict.py │ │ │ ├── test_character_encoding.py │ │ │ ├── test_download_policies.py │ │ │ ├── test_pulp_to_pulp.py │ │ │ ├── test_acs.py │ │ │ ├── test_download_content.py │ │ │ ├── test_repo_sizes.py │ │ │ └── test_auto_publish.py │ │ ├── sign-metadata.sh │ │ └── utils.py │ └── sample-rpm-0-0.x86_64.rpm ├── app │ ├── kickstart │ │ └── __init__.py │ ├── management │ │ ├── __init__.py │ │ └── commands │ │ │ ├── __init__.py │ │ │ └── rpm-trim-changelogs.py │ ├── migrations │ │ ├── __init__.py │ │ ├── 0020_remove_updatecollection_m2m.py │ │ ├── 0042_alter_repometadatafile_data_type.py │ │ ├── 0002_updaterecord_reboot_suggested.py │ │ ├── 0031_modulemd_static_context.py │ │ ├── 0011_rpmremote_sles_auth_token.py │ │ ├── 0010_revision_null_redo.py │ │ ├── 0053_rpmdistribution_generate_repo_config.py │ │ ├── 0009_revision_null.py │ │ ├── 0014_rpmrepository_package_retention_policy.py │ │ ├── 0028_rpmrepository_last_sync_repomd_cheksum.py │ │ ├── 0021_rename_updatecollection_update_record.py │ │ ├── 0056_remove_rpmpublication_sqlite_metadata_and_more.py │ │ ├── 0065_alter_package_options.py │ │ ├── 0025_remove_orphaned_subrepos.py │ │ ├── 0060_rpmpublication_compression_type_empty.py │ │ ├── 0024_change_subrepo_relation_properties.py │ │ ├── 0018_updatecollection__update_record.py │ │ ├── 0004_add_metadata_signing_service_fk.py │ │ ├── 0055_add_repo_config_field.py │ │ ├── 0026_add_gpgcheck_options.py │ │ ├── 0037_DATA_remove_rpmrepository_sub_repo.py │ │ ├── 0029_rpmpublication_sqlite_metadata.py │ │ ├── 0022_add_collections_related_name.py │ │ ├── 0059_rpmpublication_compression_type_and_more.py │ │ ├── 0054_remove_gpg_fields.py │ │ ├── 0063_rpmpublication_layout_rpmrepository_layout.py │ │ ├── 0012_remove_pkg_group_env_cat_related_pkgs.py │ │ ├── 0023_increase_distribution_release_short.py │ │ ├── 0040_rpmalternatecontentsource.py │ │ ├── 0032_ulnremote.py │ │ ├── 0016_dist_tree_nofk.py │ │ ├── 0005_optimize_sync.py │ │ ├── 0036_checksum_type.py │ │ ├── 0006_opensuse_support.py │ │ ├── 0038_fix_sync_optimization.py │ │ ├── 0015_repo_metadata.py │ │ ├── 0030_DATA_fix_updaterecord.py │ │ ├── 0058_alter_addon_repository_alter_variant_repository.py │ │ ├── 0064_remove_rpmrepository_original_checksum_types_and_more.py │ │ ├── 0048_artifacts_dependencies_fix.py │ │ ├── 0061_fix_modulemd_defaults_digest.py │ │ ├── 0049_profiles_fix.py │ │ ├── 0007_checksum_types.py │ │ ├── 0039_disttree_digest.py │ │ ├── 0035_fix_auto_publish.py │ │ ├── 0041_modulemdobsolete.py │ │ ├── 0062_rpmpackagesigningservice_and_more.py │ │ ├── 0027_checksum_null.py │ │ ├── 0057_rpmpublication_checksum_type_and_more.py │ │ ├── 0008_advisory_pkg_sumtype_as_int.py │ │ ├── 0034_auto_publish.py │ │ ├── 0052_modulemd_digest.py │ │ ├── 0017_merge_advisory_collections.py │ │ ├── 0003_DATA_incorrect_json.py │ │ ├── 0046_rbac_perms.py │ │ ├── 0047_modulemd_datefield.py │ │ ├── 0044_noartifact_modules.py │ │ ├── 0045_modulemd_fields.py │ │ └── 0019_migrate_updatecollection_data.py │ ├── tasks │ │ ├── __init__.py │ │ └── signing.py │ ├── __init__.py │ ├── schema │ │ ├── __init__.py │ │ ├── copy_config.json │ │ └── modulemd.json │ ├── urls.py │ ├── models │ │ ├── acs.py │ │ ├── __init__.py │ │ ├── custom_metadata.py │ │ └── content.py │ ├── viewsets │ │ ├── custom_metadata.py │ │ ├── distribution.py │ │ ├── __init__.py │ │ ├── advisory.py │ │ └── prune.py │ ├── settings.py │ ├── serializers │ │ ├── acs.py │ │ ├── __init__.py │ │ ├── custom_metadata.py │ │ └── prune.py │ ├── replica.py │ ├── access_policy.py │ ├── comps.py │ ├── exceptions.py │ └── fields.py └── __init__.py ├── CHANGES ├── .gitignore ├── +datarepair-dry-run.misc └── .TEMPLATE.md ├── test_requirements.txt ├── unittest_requirements.txt ├── dev_requirements.txt ├── .github ├── ISSUE_TEMPLATE │ ├── task.md │ ├── feature_request.md │ └── bug_report.md ├── workflows │ ├── scripts │ │ ├── secrets.py │ │ ├── push_branch_and_tag_to_github.sh │ │ ├── check_commit.sh │ │ ├── utils.sh │ │ ├── release.sh │ │ ├── build_ruby_client.sh │ │ ├── post_before_script.sh │ │ ├── build_python_client.sh │ │ ├── before_script.sh │ │ ├── update_backport_labels.py │ │ ├── pre_before_install.sh │ │ ├── stage-changelog-for-default-branch.py │ │ └── before_install.sh │ ├── codeql-analysis.yml │ ├── update-labels.yml │ ├── docs.yml │ ├── release.yml │ ├── lint.yml │ ├── nightly.yml │ └── pr_checks.yml └── stale.yml ├── .pep8speaks.yml ├── functest_requirements.txt ├── doc_requirements.txt ├── lint_requirements.txt ├── MANIFEST.in ├── COPYRIGHT ├── README.rst ├── .gitignore ├── .flake8 ├── releasing.md ├── COMMITMENT └── CONTRIBUTING.rst /.ci/assets/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docs/admin/learn/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docs/user/reference/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pulp_rpm/tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pulp_rpm/tests/unit/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /CHANGES/.gitignore: -------------------------------------------------------------------------------- 1 | !.gitignore 2 | -------------------------------------------------------------------------------- /pulp_rpm/app/kickstart/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pulp_rpm/app/management/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pulp_rpm/app/migrations/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pulp_rpm/tests/performance/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pulp_rpm/app/management/commands/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pulp_rpm/tests/functional/content_handler/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pulp_rpm/tests/functional/__init__.py: -------------------------------------------------------------------------------- 1 | """Tests for file plugin.""" 2 | -------------------------------------------------------------------------------- /.ci/assets/release_requirements.txt: -------------------------------------------------------------------------------- 1 | bump-my-version 2 | gitpython 3 | towncrier 4 | -------------------------------------------------------------------------------- /pulp_rpm/__init__.py: -------------------------------------------------------------------------------- 1 | default_app_config = "pulp_rpm.app.PulpRpmPluginAppConfig" 2 | -------------------------------------------------------------------------------- /pulp_rpm/tests/functional/api/__init__.py: -------------------------------------------------------------------------------- 1 | """Tests that communicate with file plugin via the v3 API.""" 2 | -------------------------------------------------------------------------------- /test_requirements.txt: -------------------------------------------------------------------------------- 1 | # All test requirements 2 | -r functest_requirements.txt 3 | -r unittest_requirements.txt 4 | -------------------------------------------------------------------------------- /pulp_rpm/tests/sample-rpm-0-0.x86_64.rpm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pulp/pulp_rpm/HEAD/pulp_rpm/tests/sample-rpm-0-0.x86_64.rpm -------------------------------------------------------------------------------- /unittest_requirements.txt: -------------------------------------------------------------------------------- 1 | # Unit test requirements 2 | pytest<8 3 | asynctest 4 | mock 5 | pytest-django 6 | pytest-custom_exit_code 7 | -------------------------------------------------------------------------------- /dev_requirements.txt: -------------------------------------------------------------------------------- 1 | check-manifest 2 | coverage 3 | flake8 4 | flake8-black 5 | flake8-docstrings 6 | flake8-tuple 7 | flake8-quotes 8 | requests 9 | -------------------------------------------------------------------------------- /.ci/assets/httpie/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "default_options": [ 3 | "--ignore-stdin", 4 | "--pretty=format", 5 | "--traceback" 6 | ] 7 | } 8 | -------------------------------------------------------------------------------- /CHANGES/+datarepair-dry-run.misc: -------------------------------------------------------------------------------- 1 | Added a `--dry-run` flag to the `rpm-datarepair` management command, to be able to see affected packages without making any changes. -------------------------------------------------------------------------------- /.ci/ansible/ansible.cfg: -------------------------------------------------------------------------------- 1 | [defaults] 2 | inventory = inventory.yaml 3 | filter_plugins = filter 4 | retry_files_enabled = False 5 | transport = local 6 | nocows = 1 7 | stdout_callback = yaml 8 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/task.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: 🗒️ Task 3 | about: Documentation, CI/CD, refactors, investigations 4 | title: '' 5 | labels: Task, Triage-Needed 6 | assignees: '' 7 | 8 | --- 9 | 10 | 11 | -------------------------------------------------------------------------------- /.ci/ansible/inventory.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | all: 3 | children: 4 | containers: 5 | hosts: 6 | pulp: 7 | pulp-fixtures: 8 | minio: 9 | ci-sftp: 10 | vars: 11 | ansible_connection: docker 12 | ... 13 | -------------------------------------------------------------------------------- /.pep8speaks.yml: -------------------------------------------------------------------------------- 1 | pycodestyle: 2 | max-line-length: 100 # Default is 79 in PEP8 3 | ignore: # Errors and warnings to ignore 4 | exclude: 5 | - "./docs/*" 6 | - "*/build/*" 7 | - "*/migrations/*" 8 | 9 | # E401: multiple imports on one line 10 | -------------------------------------------------------------------------------- /functest_requirements.txt: -------------------------------------------------------------------------------- 1 | dictdiffer 2 | django # TODO: test_sync.py has a dependency on date parsing functions 3 | lxml 4 | productmd>=1.25 5 | pydantic 6 | pyyaml 7 | pytest<8 8 | pytest-xdist 9 | pytest-timeout 10 | pytest-custom_exit_code 11 | pyzstd 12 | requests 13 | xmltodict -------------------------------------------------------------------------------- /pulp_rpm/app/tasks/__init__.py: -------------------------------------------------------------------------------- 1 | from .publishing import publish # noqa 2 | from .synchronizing import synchronize # noqa 3 | from .signing import sign_and_create # noqa 4 | from .copy import copy_content # noqa 5 | from .comps import upload_comps # noqa 6 | from .prune import prune_packages # noqa 7 | -------------------------------------------------------------------------------- /pulp_rpm/tests/unit/test_models.py: -------------------------------------------------------------------------------- 1 | from django.test import TestCase 2 | 3 | 4 | class TestNothing(TestCase): 5 | """Test Nothing (placeholder).""" 6 | 7 | def test_nothing_at_all(self): 8 | """Test that the tests are running and that's it.""" 9 | self.assertTrue(True) 10 | -------------------------------------------------------------------------------- /doc_requirements.txt: -------------------------------------------------------------------------------- 1 | # WARNING: DO NOT EDIT! 2 | # 3 | # This file was generated by plugin_template, and is managed by it. Please use 4 | # './plugin-template --github pulp_rpm' to update this file. 5 | # 6 | # For more info visit https://github.com/pulp/plugin_template 7 | towncrier 8 | pulp-docs @ git+https://github.com/pulp/pulp-docs@main 9 | -------------------------------------------------------------------------------- /docs/dev/guides/bindings.md: -------------------------------------------------------------------------------- 1 | # Client Bindings 2 | 3 | Client bindings are language specific libraries that implements the Pulp API. 4 | 5 | Refer to the [pulp-openapi-generator guide] to learn about how to generate them for a given Pulp installation. 6 | 7 | [pulp-openapi-generator guide]: site:pulp-openapi-generator/docs/user/guides/generate-bindings/ 8 | -------------------------------------------------------------------------------- /docs/dev/index.md: -------------------------------------------------------------------------------- 1 | # Welcome to Pulp RPM for developers! 2 | 3 | Here you'll find information useful for the RPM plugin developers. 4 | 5 | If you just got here, consider exploring Pulpcore's [Developer Manual](site:pulpcore/docs/dev/), as it provides the common ground for developers for contributing to docs, to code and getting basic background on plugin development. 6 | 7 | -------------------------------------------------------------------------------- /pulp_rpm/app/__init__.py: -------------------------------------------------------------------------------- 1 | from pulpcore.plugin import PulpPluginAppConfig 2 | 3 | 4 | class PulpRpmPluginAppConfig(PulpPluginAppConfig): 5 | """ 6 | Entry point for pulp_rpm plugin. 7 | """ 8 | 9 | name = "pulp_rpm.app" 10 | label = "rpm" 11 | version = "3.34.0.dev" 12 | python_package_name = "pulp-rpm" 13 | domain_compatible = True 14 | -------------------------------------------------------------------------------- /lint_requirements.txt: -------------------------------------------------------------------------------- 1 | # WARNING: DO NOT EDIT! 2 | # 3 | # This file was generated by plugin_template, and is managed by it. Please use 4 | # './plugin-template --github pulp_rpm' to update this file. 5 | # 6 | # For more info visit https://github.com/pulp/plugin_template 7 | 8 | black==24.3.0 9 | bump-my-version 10 | check-manifest 11 | flake8 12 | flake8-black 13 | packaging 14 | yamllint 15 | -------------------------------------------------------------------------------- /pulp_rpm/app/schema/__init__.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | 4 | location = os.path.dirname(os.path.realpath(__file__)) 5 | 6 | with open(os.path.join(location, "copy_config.json")) as copy_config_json: 7 | COPY_CONFIG_SCHEMA = json.load(copy_config_json) 8 | 9 | with open(os.path.join(location, "modulemd.json")) as modulemd_json: 10 | MODULEMD_SCHEMA = json.load(modulemd_json) 11 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include CHANGES.md 2 | include COMMITMENT 3 | include COPYRIGHT 4 | include coverage.md 5 | include functest_requirements.txt 6 | include LICENSE 7 | include pulp_rpm/app/schema/* 8 | include pulp_rpm/tests/functional/sign-metadata.sh 9 | include pulp_rpm/tests/sample-rpm-0-0.x86_64.rpm 10 | include pyproject.toml 11 | include test_requirements.txt 12 | include unittest_requirements.txt 13 | exclude releasing.md 14 | -------------------------------------------------------------------------------- /docs/admin/index.md: -------------------------------------------------------------------------------- 1 | # Welcome to Pulp RPM for admins! 2 | 3 | Here you'll find information about RPM-specific admin workflows. 4 | 5 | If you just got here, consider following the top [Admin Manual](site:pulpcore/#admin) links, as it provides the common ground for setting up and configuring your Pulp deployment. 6 | 7 | You may also find useful to have a look at [RPM specific Pulp settings](site:pulp_rpm/docs/admin/reference/settings/). 8 | 9 | -------------------------------------------------------------------------------- /docs/user/guides/_SUMMARY.md: -------------------------------------------------------------------------------- 1 | [//]: Learn more here about the syntax: 2 | [//]: https://pulpproject.org/pulp-docs/docs/dev/reference/markdown-cheatsheet/#ordering-files 3 | 4 | * [Get Content from Pulp](use_pulp_repo.md) 5 | * [Upload Content](upload.md) 6 | * [Modify Content](modify.md) 7 | * [Alternate Content Source](alternate-content-source.md) 8 | * [Sign Repository Metadata](metadata_signing.md) 9 | * [Sign Packages](sign-packages.md) 10 | * [Prune Packages](prune.md) 11 | 12 | -------------------------------------------------------------------------------- /pulp_rpm/app/migrations/0020_remove_updatecollection_m2m.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.14 on 2020-08-06 00:50 2 | 3 | from django.db import migrations 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ('rpm', '0019_migrate_updatecollection_data'), 10 | ] 11 | 12 | operations = [ 13 | migrations.RemoveField( 14 | model_name='updatecollection', 15 | name='update_record', 16 | ), 17 | ] 18 | -------------------------------------------------------------------------------- /pulp_rpm/app/migrations/0042_alter_repometadatafile_data_type.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.2.13 on 2022-05-02 07:36 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ('rpm', '0041_modulemdobsolete'), 10 | ] 11 | 12 | operations = [ 13 | migrations.AlterField( 14 | model_name='repometadatafile', 15 | name='data_type', 16 | field=models.TextField(), 17 | ), 18 | ] 19 | -------------------------------------------------------------------------------- /pulp_rpm/app/migrations/0002_updaterecord_reboot_suggested.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.7 on 2019-11-18 11:53 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ('rpm', '0001_initial'), 10 | ] 11 | 12 | operations = [ 13 | migrations.AddField( 14 | model_name='updaterecord', 15 | name='reboot_suggested', 16 | field=models.BooleanField(default=False), 17 | ), 18 | ] 19 | -------------------------------------------------------------------------------- /pulp_rpm/app/migrations/0031_modulemd_static_context.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.20 on 2021-05-12 11:50 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ('rpm', '0030_DATA_fix_updaterecord'), 10 | ] 11 | 12 | operations = [ 13 | migrations.AddField( 14 | model_name='modulemd', 15 | name='static_context', 16 | field=models.BooleanField(null=True), 17 | ), 18 | ] 19 | -------------------------------------------------------------------------------- /pulp_rpm/app/migrations/0011_rpmremote_sles_auth_token.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.12 on 2020-06-16 12:03 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ('rpm', '0010_revision_null_redo'), 10 | ] 11 | 12 | operations = [ 13 | migrations.AddField( 14 | model_name='rpmremote', 15 | name='sles_auth_token', 16 | field=models.CharField(max_length=512, null=True), 17 | ), 18 | ] 19 | -------------------------------------------------------------------------------- /pulp_rpm/app/migrations/0010_revision_null_redo.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.12 on 2020-05-06 15:15 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ('rpm', '0009_revision_null'), 10 | ] 11 | 12 | operations = [ 13 | migrations.AlterField( 14 | model_name='rpmrepository', 15 | name='last_sync_revision_number', 16 | field=models.CharField(max_length=20, null=True), 17 | ), 18 | ] 19 | -------------------------------------------------------------------------------- /pulp_rpm/app/migrations/0053_rpmdistribution_generate_repo_config.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 4.2.4 on 2023-09-06 14:04 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ("rpm", "0052_modulemd_digest"), 10 | ] 11 | 12 | operations = [ 13 | migrations.AddField( 14 | model_name="rpmdistribution", 15 | name="generate_repo_config", 16 | field=models.BooleanField(default=False), 17 | ), 18 | ] 19 | -------------------------------------------------------------------------------- /pulp_rpm/app/migrations/0009_revision_null.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.12 on 2020-05-06 15:15 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ('rpm', '0008_advisory_pkg_sumtype_as_int'), 10 | ] 11 | 12 | operations = [ 13 | migrations.AlterField( 14 | model_name='rpmrepository', 15 | name='last_sync_revision_number', 16 | field=models.CharField(max_length=20, null=True), 17 | ), 18 | ] 19 | -------------------------------------------------------------------------------- /pulp_rpm/tests/functional/sign-metadata.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | FILE_PATH=$1 4 | SIGNATURE_PATH="$1.asc" 5 | 6 | GPG_KEY_ID="pulp-fixture-signing-key" 7 | 8 | # Create a detached signature 9 | gpg --quiet --batch --homedir ~/.gnupg/ --detach-sign --local-user "${GPG_KEY_ID}" \ 10 | --armor --output ${SIGNATURE_PATH} ${FILE_PATH} 11 | 12 | # Check the exit status 13 | STATUS=$? 14 | if [[ ${STATUS} -eq 0 ]]; then 15 | echo {\"file\": \"${FILE_PATH}\", \"signature\": \"${SIGNATURE_PATH}\"} 16 | else 17 | exit ${STATUS} 18 | fi 19 | -------------------------------------------------------------------------------- /pulp_rpm/app/migrations/0014_rpmrepository_package_retention_policy.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.13 on 2020-06-24 19:32 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ('rpm', '0013_RAW_rpm_evr_extension'), 10 | ] 11 | 12 | operations = [ 13 | migrations.AddField( 14 | model_name='rpmrepository', 15 | name='retain_package_versions', 16 | field=models.PositiveIntegerField(default=0), 17 | ), 18 | ] 19 | -------------------------------------------------------------------------------- /pulp_rpm/app/migrations/0028_rpmrepository_last_sync_repomd_cheksum.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.16 on 2020-10-15 11:55 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ('rpm', '0027_checksum_null'), 10 | ] 11 | 12 | operations = [ 13 | migrations.AddField( 14 | model_name='rpmrepository', 15 | name='last_sync_repomd_checksum', 16 | field=models.CharField(max_length=64, null=True), 17 | ), 18 | ] 19 | -------------------------------------------------------------------------------- /.github/workflows/scripts/secrets.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | import sys 4 | 5 | secrets = json.loads(sys.argv[1]) 6 | for key, value in secrets.items(): 7 | print(f"Setting {key} ...") 8 | lines = len(value.split("\n")) 9 | if lines > 1: 10 | os.system(f"/bin/bash -c \"echo '{key}<> $GITHUB_ENV\"") 11 | os.system(f"/bin/bash -c \"echo '{value}' >> $GITHUB_ENV\"") 12 | os.system("/bin/bash -c \"echo 'EOF' >> $GITHUB_ENV\"") 13 | else: 14 | os.system(f"/bin/bash -c \"echo '{key}={value}' >> $GITHUB_ENV\"") 15 | -------------------------------------------------------------------------------- /.github/workflows/scripts/push_branch_and_tag_to_github.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # WARNING: DO NOT EDIT! 4 | # 5 | # This file was generated by plugin_template, and is managed by it. Please use 6 | # './plugin-template --github pulp_rpm' to update this file. 7 | # 8 | # For more info visit https://github.com/pulp/plugin_template 9 | 10 | set -eu 11 | 12 | BRANCH_NAME="$(echo "$GITHUB_REF" | sed -rn 's/refs\/heads\/(.*)/\1/p')" 13 | 14 | remote_repo="https://pulpbot:${RELEASE_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" 15 | 16 | git push "${remote_repo}" "$BRANCH_NAME" "$1" 17 | -------------------------------------------------------------------------------- /COPYRIGHT: -------------------------------------------------------------------------------- 1 | Copyright © 2014 Pulp Project developers. 2 | 3 | This software is licensed to you under the GNU General Public 4 | License as published by the Free Software Foundation; either version 5 | 2 of the License (GPLv2) or (at your option) any later version. 6 | There is NO WARRANTY for this software, express or implied, 7 | including the implied warranties of MERCHANTABILITY, 8 | NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should 9 | have received a copy of GPLv2 along with this software; if not, see 10 | http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. 11 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | ``pulp_rpm`` Plugin 2 | =================== 3 | 4 | .. figure:: https://github.com/pulp/pulp_rpm/actions/workflows/nightly.yml/badge.svg?branch=main 5 | :alt: Rpm Nightly CI/CD 6 | 7 | This is the ``pulp_rpm`` Plugin for `Pulp Project 8 | 3.0+ `__. This plugin provides support for RPM family 9 | content types, similar to the ``pulp_rpm`` plugin for Pulp 2. 10 | 11 | For more information, please see the `documentation 12 | `_ or the `Pulp project page 13 | `_. 14 | -------------------------------------------------------------------------------- /pulp_rpm/app/migrations/0021_rename_updatecollection_update_record.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.14 on 2020-08-06 00:54 2 | 3 | from django.db import migrations, models 4 | import django.db.models.deletion 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ('rpm', '0020_remove_updatecollection_m2m'), 11 | ] 12 | 13 | operations = [ 14 | migrations.RenameField( 15 | model_name='updatecollection', 16 | old_name='_update_record', 17 | new_name='update_record', 18 | ), 19 | ] 20 | -------------------------------------------------------------------------------- /.ci/scripts/check_gettext.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # WARNING: DO NOT EDIT! 4 | # 5 | # This file was generated by plugin_template, and is managed by it. Please use 6 | # './plugin-template --github pulp_rpm' to update this file. 7 | # 8 | # For more info visit https://github.com/pulp/plugin_template 9 | 10 | # make sure this script runs at the repo root 11 | cd "$(dirname "$(realpath -e "$0")")"/../.. 12 | 13 | set -uv 14 | 15 | MATCHES=$(grep -n -r --include \*.py "_(f") 16 | 17 | if [ $? -ne 1 ]; then 18 | printf "\nERROR: Detected mix of f-strings and gettext:\n" 19 | echo "$MATCHES" 20 | exit 1 21 | fi 22 | -------------------------------------------------------------------------------- /.ci/assets/ci_constraints.txt: -------------------------------------------------------------------------------- 1 | # Pulpcore versions without the openapi command do no longer work in the CI 2 | pulpcore>=3.21.30,!=3.23.*,!=3.24.*,!=3.25.*,!=3.26.*,!=3.27.*,!=3.29.*,!=3.30.*,!=3.31.*,!=3.32.*,!=3.33.*,!=3.34.*,!=3.35.*,!=3.36.*,!=3.37.*,!=3.38.*,!=3.40.*,!=3.41.*,!=3.42.*,!=3.43.*,!=3.44.*,!=3.45.*,!=3.46.*,!=3.47.*,!=3.48.*,!=3.50.*,!=3.51.*,!=3.52.*,!=3.53.*,!=3.54.* 3 | 4 | 5 | tablib!=3.6.0 6 | # 3.6.0: This release introduced a regression removing the "html" optional dependency. 7 | 8 | 9 | multidict!=6.3.0 10 | # This release failed the lower bounds test for some case sensitivity in CIMultiDict. 11 | -------------------------------------------------------------------------------- /.github/workflows/scripts/check_commit.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # WARNING: DO NOT EDIT! 4 | # 5 | # This file was generated by plugin_template, and is managed by it. Please use 6 | # './plugin-template --github pulp_rpm' to update this file. 7 | # 8 | # For more info visit https://github.com/pulp/plugin_template 9 | 10 | # make sure this script runs at the repo root 11 | cd "$(dirname "$(realpath -e "$0")")/../../.." 12 | 13 | set -euv 14 | 15 | for SHA in $(curl -H "Authorization: token $GITHUB_TOKEN" "$GITHUB_CONTEXT" | jq -r '.[].sha') 16 | do 17 | python3 .ci/scripts/validate_commit_message.py "$SHA" 18 | done 19 | -------------------------------------------------------------------------------- /pulp_rpm/app/migrations/0056_remove_rpmpublication_sqlite_metadata_and_more.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 4.2.7 on 2023-11-22 18:48 2 | 3 | from django.db import migrations 4 | 5 | 6 | class Migration(migrations.Migration): 7 | dependencies = [ 8 | ("rpm", "0055_add_repo_config_field"), 9 | ] 10 | 11 | operations = [ 12 | migrations.RemoveField( 13 | model_name="rpmpublication", 14 | name="sqlite_metadata", 15 | ), 16 | migrations.RemoveField( 17 | model_name="rpmrepository", 18 | name="sqlite_metadata", 19 | ), 20 | ] 21 | -------------------------------------------------------------------------------- /pulp_rpm/app/migrations/0065_alter_package_options.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 4.2.10 on 2025-07-01 20:29 2 | 3 | from django.db import migrations 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ('rpm', '0064_remove_rpmrepository_original_checksum_types_and_more'), 10 | ] 11 | 12 | operations = [ 13 | migrations.AlterModelOptions( 14 | name='package', 15 | options={'default_related_name': '%(app_label)s_%(model_name)s', 'permissions': [('upload_rpm_packages', 'Can upload RPM packages using synchronous API.')]}, 16 | ), 17 | ] 18 | -------------------------------------------------------------------------------- /pulp_rpm/app/migrations/0025_remove_orphaned_subrepos.py: -------------------------------------------------------------------------------- 1 | from django.db import migrations 2 | 3 | def delete_orphan_subrepos(apps, schema): 4 | """ 5 | Remove subrepos which don't belong to any variant or addon. 6 | """ 7 | RpmRepository = apps.get_model("rpm", "RpmRepository") 8 | RpmRepository.objects.filter(addons=None,variants=None,sub_repo=True).delete() 9 | 10 | class Migration(migrations.Migration): 11 | 12 | dependencies = [ 13 | ('rpm', '0024_change_subrepo_relation_properties'), 14 | ] 15 | 16 | operations = [ 17 | migrations.RunPython(delete_orphan_subrepos) 18 | ] 19 | -------------------------------------------------------------------------------- /pulp_rpm/app/urls.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | from django.urls import path 3 | 4 | from .viewsets import CopyViewSet, CompsXmlViewSet, PrunePackagesViewSet 5 | 6 | if settings.DOMAIN_ENABLED: 7 | V3_API_ROOT = settings.V3_DOMAIN_API_ROOT_NO_FRONT_SLASH 8 | else: 9 | V3_API_ROOT = settings.V3_API_ROOT_NO_FRONT_SLASH 10 | 11 | urlpatterns = [ 12 | path(f"{V3_API_ROOT}rpm/copy/", CopyViewSet.as_view({"post": "create"})), 13 | path(f"{V3_API_ROOT}rpm/comps/", CompsXmlViewSet.as_view({"post": "create"})), 14 | path(f"{V3_API_ROOT}rpm/prune/", PrunePackagesViewSet.as_view({"post": "prune_packages"})), 15 | ] 16 | -------------------------------------------------------------------------------- /pulp_rpm/app/migrations/0060_rpmpublication_compression_type_empty.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 4.2.5 on 2023-11-07 03:51 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | def set_publication_checksum(apps, schema_editor): 7 | RpmPublication = apps.get_model("rpm", "RpmPublication") 8 | RpmPublication.objects.filter(checksum_type="").update(checksum_type="unknown") 9 | 10 | 11 | class Migration(migrations.Migration): 12 | 13 | dependencies = [ 14 | ('rpm', '0059_rpmpublication_compression_type_and_more'), 15 | ] 16 | 17 | operations = [ 18 | migrations.RunPython(set_publication_checksum), 19 | ] 20 | -------------------------------------------------------------------------------- /pulp_rpm/app/migrations/0024_change_subrepo_relation_properties.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.15 on 2020-09-04 16:17 2 | 3 | from django.db import migrations, models 4 | import django.db.models.deletion 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ('rpm', '0023_increase_distribution_release_short'), 11 | ] 12 | 13 | operations = [ 14 | migrations.AlterField( 15 | model_name='variant', 16 | name='repository', 17 | field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.PROTECT, related_name='variants', to='core.Repository'), 18 | ), 19 | ] 20 | -------------------------------------------------------------------------------- /.ci/scripts/update_github.py: -------------------------------------------------------------------------------- 1 | # WARNING: DO NOT EDIT! 2 | # 3 | # This file was generated by plugin_template, and is managed by it. Please use 4 | # './plugin-template --github pulp_rpm' to update this file. 5 | # 6 | # For more info visit https://github.com/pulp/plugin_template 7 | 8 | import os 9 | from github import Github 10 | 11 | g = Github(os.environ.get("GITHUB_TOKEN")) 12 | repo = g.get_repo("pulp/pulp_rpm") 13 | 14 | GH_ISSUES = os.environ.get("GH_ISSUES") 15 | 16 | for issue in GH_ISSUES.split(","): 17 | issue = repo.get_issue(int(issue)) 18 | if issue.state != "closed": 19 | print(f"Closing issue: {issue.number}") 20 | issue.edit(state="closed") 21 | -------------------------------------------------------------------------------- /pulp_rpm/app/migrations/0018_updatecollection__update_record.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.14 on 2020-08-03 18:58 2 | 3 | from django.db import migrations, models 4 | import django.db.models.deletion 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ('rpm', '0017_merge_advisory_collections'), 11 | ] 12 | 13 | operations = [ 14 | migrations.AddField( 15 | model_name='updatecollection', 16 | name='_update_record', 17 | field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='_collections', to='rpm.UpdateRecord'), 18 | ), 19 | ] 20 | -------------------------------------------------------------------------------- /pulp_rpm/app/schema/copy_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://json-schema.org/draft-07/schema#", 3 | "title": "CopyConfig", 4 | "description": "Config for copying content between repos", 5 | "type": "array", 6 | "minItems": 1, 7 | "items": { 8 | "type": "object", 9 | "additionProperties": false, 10 | "required": [ "source_repo_version", "dest_repo" ], 11 | "properties": { 12 | "source_repo_version": { "type": "string" }, 13 | "dest_repo": { "type": "string" }, 14 | "dest_base_version": { "type": "integer" }, 15 | "content": { 16 | "type": "array", 17 | "items": { "type": "string" } 18 | } 19 | } 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /pulp_rpm/app/migrations/0004_add_metadata_signing_service_fk.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 4.2.17 on 2024-12-05 18:54 2 | 3 | from django.db import migrations, models 4 | import django.db.models.deletion 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ('rpm', '0003_DATA_incorrect_json'), 11 | ] 12 | 13 | operations = [ 14 | migrations.AddField( 15 | model_name='rpmrepository', 16 | name='metadata_signing_service', 17 | field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='rpm_rpmrepository', to='core.asciiarmoreddetachedsigningservice'), 18 | ), 19 | ] 20 | -------------------------------------------------------------------------------- /pulp_rpm/app/migrations/0055_add_repo_config_field.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 4.2.4 on 2023-09-08 14:45 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ("rpm", "0054_remove_gpg_fields"), 10 | ] 11 | 12 | operations = [ 13 | migrations.AddField( 14 | model_name="rpmpublication", 15 | name="repo_config", 16 | field=models.JSONField(default=dict), 17 | ), 18 | migrations.AddField( 19 | model_name="rpmrepository", 20 | name="repo_config", 21 | field=models.JSONField(default=dict), 22 | ), 23 | ] 24 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.py[co] 2 | 3 | # Packages 4 | *.egg 5 | *.egg-info 6 | dist 7 | build 8 | pip-wheel-metadata/ 9 | eggs 10 | parts 11 | bin 12 | var 13 | sdist 14 | develop-eggs 15 | .installed.cfg 16 | 17 | # Installer logs 18 | pip-log.txt 19 | 20 | # Unit test / coverage reports 21 | .coverage 22 | .tox 23 | 24 | #Translations 25 | *.mo 26 | 27 | # Eclipse 28 | .project 29 | .pydevproject 30 | .settings 31 | 32 | # Mr Developer 33 | .mr.developer.cfg 34 | 35 | # Ninja IDE 36 | *.nja 37 | 38 | # PyCharm 39 | .idea 40 | cover 41 | 42 | #VScode 43 | .vscode/ 44 | 45 | # Ninja IDE 46 | *.nja 47 | 48 | # Sphinx 49 | docs/_build 50 | docs/_static/api.json 51 | 52 | # Vim 53 | *.swp 54 | 55 | *.ropeproject 56 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: ✨ Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: Feature, Triage-Needed 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context about the feature request here. 21 | -------------------------------------------------------------------------------- /pulp_rpm/tests/functional/api/test_prn.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | 4 | @pytest.mark.parallel 5 | def test_prn_schema(pulp_openapi_schema): 6 | """Test that PRN is a part of every serializer with a pulp_href.""" 7 | failed = [] 8 | for name, schema in pulp_openapi_schema["components"]["schemas"].items(): 9 | if name.endswith("Response"): 10 | if "pulp_href" in schema["properties"]: 11 | if "prn" in schema["properties"]: 12 | prn_schema = schema["properties"]["prn"] 13 | if prn_schema["type"] == "string" and prn_schema["readOnly"]: 14 | continue 15 | failed.append(name) 16 | 17 | assert len(failed) == 0 18 | -------------------------------------------------------------------------------- /pulp_rpm/app/migrations/0026_add_gpgcheck_options.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.16 on 2020-10-09 15:13 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ('rpm', '0025_remove_orphaned_subrepos'), 10 | ] 11 | 12 | operations = [ 13 | migrations.AddField( 14 | model_name='rpmpublication', 15 | name='gpgcheck', 16 | field=models.IntegerField(choices=[(0, 0), (1, 1)], default=0), 17 | ), 18 | migrations.AddField( 19 | model_name='rpmpublication', 20 | name='repo_gpgcheck', 21 | field=models.IntegerField(choices=[(0, 0), (1, 1)], default=0), 22 | ), 23 | ] 24 | -------------------------------------------------------------------------------- /pulp_rpm/app/migrations/0037_DATA_remove_rpmrepository_sub_repo.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.24 on 2021-08-03 21:04 2 | 3 | from django.db import migrations 4 | 5 | 6 | def migrate_sub_repo_value(apps, schema_editor): 7 | RpmRepository = apps.get_model('rpm', 'RpmRepository') 8 | RpmRepository.objects.filter(sub_repo=True).update(user_hidden=True) 9 | 10 | 11 | class Migration(migrations.Migration): 12 | 13 | dependencies = [ 14 | ('rpm', '0036_checksum_type'), 15 | ] 16 | 17 | operations = [ 18 | migrations.RunPython( 19 | migrate_sub_repo_value 20 | ), 21 | migrations.RemoveField( 22 | model_name='rpmrepository', 23 | name='sub_repo', 24 | ), 25 | ] 26 | -------------------------------------------------------------------------------- /pulp_rpm/app/migrations/0029_rpmpublication_sqlite_metadata.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.17 on 2020-12-14 22:12 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | def set_true(apps, schema_editor): 7 | Publication = apps.get_model("rpm", "RpmPublication") 8 | Publication.objects.update(sqlite_metadata=True) 9 | 10 | class Migration(migrations.Migration): 11 | 12 | dependencies = [ 13 | ('rpm', '0028_rpmrepository_last_sync_repomd_cheksum'), 14 | ] 15 | 16 | operations = [ 17 | migrations.AddField( 18 | model_name='rpmpublication', 19 | name='sqlite_metadata', 20 | field=models.BooleanField(default=False), 21 | ), 22 | migrations.RunPython(set_true) 23 | ] 24 | -------------------------------------------------------------------------------- /pulp_rpm/app/models/acs.py: -------------------------------------------------------------------------------- 1 | from logging import getLogger 2 | 3 | from pulpcore.plugin.models import AlternateContentSource, AutoAddObjPermsMixin 4 | from pulp_rpm.app.models import RpmRemote 5 | 6 | 7 | log = getLogger(__name__) 8 | 9 | 10 | class RpmAlternateContentSource(AlternateContentSource, AutoAddObjPermsMixin): 11 | """ 12 | Alternate Content Source for 'RPM" content. 13 | """ 14 | 15 | TYPE = "rpm" 16 | REMOTE_TYPES = [RpmRemote] 17 | 18 | class Meta: 19 | default_related_name = "%(app_label)s_%(model_name)s" 20 | permissions = [ 21 | ("refresh_rpmalternatecontentsource", "Refresh an Alternate Content Source"), 22 | ("manage_roles_rpmalternatecontentsource", "Can manage roles on ACS"), 23 | ] 24 | -------------------------------------------------------------------------------- /.ci/scripts/check_pulpcore_imports.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # WARNING: DO NOT EDIT! 4 | # 5 | # This file was generated by plugin_template, and is managed by it. Please use 6 | # './plugin-template --github pulp_rpm' to update this file. 7 | # 8 | # For more info visit https://github.com/pulp/plugin_template 9 | 10 | # make sure this script runs at the repo root 11 | cd "$(dirname "$(realpath -e "$0")")"/../.. 12 | 13 | set -uv 14 | 15 | # check for imports not from pulpcore.plugin. exclude tests 16 | MATCHES=$(grep -n -r --include \*.py "from pulpcore.*import" . | grep -v "tests\|plugin") 17 | 18 | if [ $? -ne 1 ]; then 19 | printf "\nERROR: Detected bad imports from pulpcore:\n" 20 | echo "$MATCHES" 21 | printf "\nPlugins should import from pulpcore.plugin." 22 | exit 1 23 | fi 24 | -------------------------------------------------------------------------------- /.ci/ansible/filter/repr.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, division, print_function 2 | from packaging.version import parse as parse_version 3 | 4 | __metaclass__ = type 5 | 6 | 7 | ANSIBLE_METADATA = { 8 | "metadata_version": "1.1", 9 | "status": ["preview"], 10 | "supported_by": "community", 11 | } 12 | 13 | 14 | def _repr_filter(value): 15 | return repr(value) 16 | 17 | 18 | def _canonical_semver_filter(value): 19 | return str(parse_version(value)) 20 | 21 | 22 | # ---- Ansible filters ---- 23 | class FilterModule(object): 24 | """Repr filter.""" 25 | 26 | def filters(self): 27 | """Filter associations.""" 28 | return { 29 | "repr": _repr_filter, 30 | "canonical_semver": _canonical_semver_filter, 31 | } 32 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: 🐛 Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: Issue, Triage-Needed 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Version** 11 | Please provide the versions of the pulpcore and pulp_rpm packages in use, and how they are installed. If you are using Pulp via Katello, please provide the Katello version. 12 | 13 | **Describe the bug** 14 | A clear and concise description of what the bug is. 15 | 16 | **To Reproduce** 17 | Steps to reproduce the behavior: 18 | 19 | **Expected behavior** 20 | A clear and concise description of what you expected to happen. 21 | 22 | **Additional context** 23 | Add any other context about the problem here. Please provide links to any previous discussions via Discourse or Bugzilla. 24 | -------------------------------------------------------------------------------- /pulp_rpm/app/migrations/0022_add_collections_related_name.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.14 on 2020-08-06 00:58 2 | 3 | from django.db import migrations, models 4 | import django.db.models.deletion 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ('rpm', '0021_rename_updatecollection_update_record'), 11 | ] 12 | 13 | operations = [ 14 | migrations.AlterField( 15 | model_name='updatecollection', 16 | name='update_record', 17 | field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='collections', to='rpm.UpdateRecord'), 18 | ), 19 | migrations.AlterUniqueTogether( 20 | name='updatecollection', 21 | unique_together={('name', 'update_record')}, 22 | ), 23 | ] 24 | -------------------------------------------------------------------------------- /pulp_rpm/app/migrations/0059_rpmpublication_compression_type_and_more.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 4.2.7 on 2023-12-12 18:37 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ('rpm', '0058_alter_addon_repository_alter_variant_repository'), 10 | ] 11 | 12 | operations = [ 13 | migrations.AddField( 14 | model_name='rpmpublication', 15 | name='compression_type', 16 | field=models.TextField(choices=[('zstd', 'zstd'), ('gz', 'gz')], null=True), 17 | ), 18 | migrations.AddField( 19 | model_name='rpmrepository', 20 | name='compression_type', 21 | field=models.TextField(choices=[('zstd', 'zstd'), ('gz', 'gz')], null=True), 22 | ), 23 | ] 24 | -------------------------------------------------------------------------------- /pulp_rpm/app/viewsets/custom_metadata.py: -------------------------------------------------------------------------------- 1 | from pulpcore.plugin.viewsets import ReadOnlyContentViewSet 2 | 3 | from pulp_rpm.app.models import RepoMetadataFile 4 | from pulp_rpm.app.serializers import RepoMetadataFileSerializer 5 | 6 | 7 | class RepoMetadataFileViewSet(ReadOnlyContentViewSet): 8 | """ 9 | RepoMetadataFile Viewset. 10 | """ 11 | 12 | endpoint_name = "repo_metadata_files" 13 | queryset = RepoMetadataFile.objects.all() 14 | serializer_class = RepoMetadataFileSerializer 15 | 16 | DEFAULT_ACCESS_POLICY = { 17 | "statements": [ 18 | { 19 | "action": ["list", "retrieve"], 20 | "principal": "authenticated", 21 | "effect": "allow", 22 | }, 23 | ], 24 | "queryset_scoping": {"function": "scope_queryset"}, 25 | } 26 | -------------------------------------------------------------------------------- /pulp_rpm/app/settings.py: -------------------------------------------------------------------------------- 1 | """ 2 | Check `Plugin Writer's Guide`_ for more details. 3 | 4 | .. _Plugin Writer's Guide: 5 | http://docs.pulpproject.org/plugins/plugin-writer/index.html 6 | """ 7 | 8 | DRF_ACCESS_POLICY = { 9 | "dynaconf_merge_unique": True, 10 | "reusable_conditions": ["pulp_rpm.app.access_policy"], 11 | } 12 | INSTALLED_APPS = ["django_readonly_field", "dynaconf_merge"] 13 | ALLOW_AUTOMATIC_UNSAFE_ADVISORY_CONFLICT_RESOLUTION = False 14 | DEFAULT_ULN_SERVER_BASE_URL = "https://linux-update.oracle.com/" 15 | KEEP_CHANGELOG_LIMIT = 10 16 | SOLVER_DEBUG_LOGS = True 17 | RPM_METADATA_USE_REPO_PACKAGE_TIME = False 18 | NOCACHE_LIST = ["repomd.xml", "repomd.xml.asc", "repomd.xml.key"] 19 | PRUNE_WORKERS_MAX = 5 20 | # workaround for: https://github.com/pulp/pulp_rpm/issues/4125 21 | SPECTACULAR_SETTINGS__OAS_VERSION = "3.0.1" 22 | -------------------------------------------------------------------------------- /pulp_rpm/app/migrations/0054_remove_gpg_fields.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 4.2.4 on 2023-09-08 14:45 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ("rpm", "0053_rpmdistribution_generate_repo_config"), 10 | ] 11 | 12 | operations = [ 13 | migrations.RemoveField( 14 | model_name="rpmpublication", 15 | name="gpgcheck", 16 | ), 17 | migrations.RemoveField( 18 | model_name="rpmpublication", 19 | name="repo_gpgcheck", 20 | ), 21 | migrations.RemoveField( 22 | model_name="rpmrepository", 23 | name="gpgcheck", 24 | ), 25 | migrations.RemoveField( 26 | model_name="rpmrepository", 27 | name="repo_gpgcheck", 28 | ), 29 | ] 30 | -------------------------------------------------------------------------------- /pulp_rpm/app/models/__init__.py: -------------------------------------------------------------------------------- 1 | from .advisory import ( # noqa 2 | UpdateCollection, 3 | UpdateCollectionPackage, 4 | UpdateRecord, 5 | UpdateReference, 6 | ) 7 | from .comps import PackageCategory, PackageEnvironment, PackageGroup, PackageLangpacks # noqa 8 | from .content import RpmPackageSigningService # noqa 9 | from .custom_metadata import RepoMetadataFile # noqa 10 | from .distribution import Addon, Checksum, DistributionTree, Image, Variant # noqa 11 | from .modulemd import Modulemd, ModulemdDefaults, ModulemdObsolete # noqa 12 | from .package import Package, format_nevra, format_nevra_short, format_nvra # noqa 13 | from .repository import RpmDistribution, RpmPublication, RpmRemote, UlnRemote, RpmRepository # noqa 14 | 15 | # at the end to avoid circular import as ACS needs import RpmRemote 16 | from .acs import RpmAlternateContentSource # noqa 17 | -------------------------------------------------------------------------------- /pulp_rpm/app/migrations/0063_rpmpublication_layout_rpmrepository_layout.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 4.2.10 on 2025-02-12 19:58 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ('rpm', '0062_rpmpackagesigningservice_and_more'), 10 | ] 11 | 12 | operations = [ 13 | migrations.AddField( 14 | model_name='rpmpublication', 15 | name='layout', 16 | field=models.TextField(choices=[('nested_alphabetically', 'nested_alphabetically'), ('flat', 'flat')], null=True), 17 | ), 18 | migrations.AddField( 19 | model_name='rpmrepository', 20 | name='layout', 21 | field=models.TextField(choices=[('nested_alphabetically', 'nested_alphabetically'), ('flat', 'flat')], null=True), 22 | ), 23 | ] 24 | -------------------------------------------------------------------------------- /.ci/ansible/smash-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "pulp": { 3 | "auth": [ 4 | "admin", 5 | "password" 6 | ], 7 | "selinux enabled": false, 8 | "version": "3", 9 | "aiohttp_fixtures_origin": "127.0.0.1" 10 | }, 11 | "hosts": [ 12 | { 13 | "hostname": "pulp", 14 | "roles": { 15 | "api": { 16 | "port": 443, 17 | "scheme": "https", 18 | "service": "nginx" 19 | }, 20 | "content": { 21 | "port": 443, 22 | "scheme": "https", 23 | "service": "pulp_content_app" 24 | }, 25 | "pulp resource manager": {}, 26 | "pulp workers": {}, 27 | "redis": {}, 28 | "shell": { 29 | "transport": "local" 30 | } 31 | } 32 | } 33 | ], 34 | "custom": { 35 | "fixtures_origin": "http://pulp-fixtures:8080/" 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /.github/workflows/scripts/utils.sh: -------------------------------------------------------------------------------- 1 | # This file is meant to be sourced by ci-scripts 2 | 3 | # WARNING: DO NOT EDIT! 4 | # 5 | # This file was generated by plugin_template, and is managed by it. Please use 6 | # './plugin-template --github pulp_rpm' to update this file. 7 | # 8 | # For more info visit https://github.com/pulp/plugin_template 9 | 10 | PULP_CI_CONTAINER=pulp 11 | 12 | # Run a command 13 | cmd_prefix() { 14 | docker exec "$PULP_CI_CONTAINER" "$@" 15 | } 16 | 17 | # Run a command as the limited pulp user 18 | cmd_user_prefix() { 19 | docker exec -u pulp "$PULP_CI_CONTAINER" "$@" 20 | } 21 | 22 | # Run a command, and pass STDIN 23 | cmd_stdin_prefix() { 24 | docker exec -i "$PULP_CI_CONTAINER" "$@" 25 | } 26 | 27 | # Run a command as the lmited pulp user, and pass STDIN 28 | cmd_user_stdin_prefix() { 29 | docker exec -i -u pulp "$PULP_CI_CONTAINER" "$@" 30 | } 31 | -------------------------------------------------------------------------------- /pulp_rpm/app/migrations/0012_remove_pkg_group_env_cat_related_pkgs.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.13 on 2020-06-30 12:57 2 | 3 | from django.db import migrations 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ('rpm', '0011_rpmremote_sles_auth_token'), 10 | ] 11 | 12 | operations = [ 13 | migrations.RemoveField( 14 | model_name='packagecategory', 15 | name='packagegroups', 16 | ), 17 | migrations.RemoveField( 18 | model_name='packageenvironment', 19 | name='optionalgroups', 20 | ), 21 | migrations.RemoveField( 22 | model_name='packageenvironment', 23 | name='packagegroups', 24 | ), 25 | migrations.RemoveField( 26 | model_name='packagegroup', 27 | name='related_packages', 28 | ), 29 | ] 30 | -------------------------------------------------------------------------------- /pulp_rpm/app/viewsets/distribution.py: -------------------------------------------------------------------------------- 1 | from pulpcore.plugin.viewsets import ( 2 | ReadOnlyContentViewSet, 3 | ) 4 | 5 | from pulp_rpm.app.models import ( 6 | DistributionTree, 7 | ) 8 | from pulp_rpm.app.serializers import ( 9 | DistributionTreeSerializer, 10 | ) 11 | 12 | 13 | class DistributionTreeViewSet(ReadOnlyContentViewSet): 14 | """ 15 | Distribution Tree Viewset. 16 | 17 | """ 18 | 19 | endpoint_name = "distribution_trees" 20 | queryset = DistributionTree.objects.all() 21 | serializer_class = DistributionTreeSerializer 22 | 23 | DEFAULT_ACCESS_POLICY = { 24 | "statements": [ 25 | { 26 | "action": ["list", "retrieve"], 27 | "principal": "authenticated", 28 | "effect": "allow", 29 | }, 30 | ], 31 | "queryset_scoping": {"function": "scope_queryset"}, 32 | } 33 | -------------------------------------------------------------------------------- /.ci/scripts/schema.py: -------------------------------------------------------------------------------- 1 | """ 2 | Customizing OpenAPI validation. 3 | 4 | OpenAPI requires paths to start with slashes: 5 | https://spec.openapis.org/oas/v3.0.3#patterned-fields 6 | 7 | But some pulp paths start with curly brackets e.g. {artifact_href} 8 | This script modifies drf-spectacular schema validation to accept slashes and curly brackets. 9 | """ 10 | 11 | import json 12 | from drf_spectacular.validation import JSON_SCHEMA_SPEC_PATH 13 | 14 | with open(JSON_SCHEMA_SPEC_PATH) as fh: 15 | openapi3_schema_spec = json.load(fh) 16 | 17 | properties = openapi3_schema_spec["definitions"]["Paths"]["patternProperties"] 18 | # Making OpenAPI validation to accept paths starting with / and { 19 | if "^\\/|{" not in properties: 20 | properties["^\\/|{"] = properties["^\\/"] 21 | del properties["^\\/"] 22 | 23 | with open(JSON_SCHEMA_SPEC_PATH, "w") as fh: 24 | json.dump(openapi3_schema_spec, fh) 25 | -------------------------------------------------------------------------------- /.github/workflows/scripts/release.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -eu -o pipefail 4 | 5 | BRANCH=$(git branch --show-current) 6 | 7 | if ! [[ "${BRANCH}" =~ ^[0-9]+\.[0-9]+$ ]] 8 | then 9 | echo ERROR: This is not a release branch! 10 | exit 1 11 | fi 12 | 13 | # The tail is a necessary workaround to remove the warning from the output. 14 | NEW_VERSION="$(bump-my-version show new_version --increment release | tail -n -1)" 15 | echo "Release ${NEW_VERSION}" 16 | 17 | if ! [[ "${NEW_VERSION}" == "${BRANCH}"* ]] 18 | then 19 | echo ERROR: Version does not match release branch 20 | exit 1 21 | fi 22 | 23 | towncrier build --yes --version "${NEW_VERSION}" 24 | bump-my-version bump release --commit --message "Release {new_version}" --tag --tag-name "{new_version}" --tag-message "Release {new_version}" --allow-dirty 25 | bump-my-version bump patch --commit 26 | 27 | git push origin "${BRANCH}" "${NEW_VERSION}" 28 | -------------------------------------------------------------------------------- /pulp_rpm/app/viewsets/__init__.py: -------------------------------------------------------------------------------- 1 | from .acs import RpmAlternateContentSourceViewSet # noqa 2 | from .advisory import UpdateRecordViewSet # noqa 3 | from .comps import ( # noqa 4 | CompsXmlViewSet, 5 | PackageGroupViewSet, 6 | PackageCategoryViewSet, 7 | PackageEnvironmentViewSet, 8 | PackageLangpacksViewSet, 9 | ) 10 | from .custom_metadata import RepoMetadataFileViewSet # noqa 11 | from .distribution import DistributionTreeViewSet # noqa 12 | from .modulemd import ModulemdViewSet, ModulemdDefaultsViewSet, ModulemdObsoleteViewSet # noqa 13 | from .package import PackageViewSet # noqa 14 | from .prune import PrunePackagesViewSet # noqa 15 | from .repository import ( # noqa 16 | RpmRepositoryViewSet, 17 | RpmRepositoryVersionViewSet, 18 | RpmRemoteViewSet, 19 | UlnRemoteViewSet, 20 | RpmPublicationViewSet, 21 | RpmDistributionViewSet, 22 | CopyViewSet, 23 | ) 24 | -------------------------------------------------------------------------------- /pulp_rpm/app/migrations/0023_increase_distribution_release_short.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.16 on 2020-09-18 13:35 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ('rpm', '0022_add_collections_related_name'), 10 | ] 11 | 12 | operations = [ 13 | migrations.AlterField( 14 | model_name='distributiontree', 15 | name='base_product_short', 16 | field=models.CharField(max_length=50, null=True), 17 | ), 18 | migrations.AlterField( 19 | model_name='distributiontree', 20 | name='release_short', 21 | field=models.CharField(max_length=50), 22 | ), 23 | migrations.AlterField( 24 | model_name='image', 25 | name='name', 26 | field=models.CharField(max_length=50), 27 | ), 28 | ] 29 | -------------------------------------------------------------------------------- /.github/workflows/scripts/build_ruby_client.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # This script expects all -api.json files to exist in the plugins root directory. 4 | # It produces a -ruby-client.tar file in the plugins root directory. 5 | 6 | # WARNING: DO NOT EDIT! 7 | # 8 | # This file was generated by plugin_template, and is managed by it. Please use 9 | # './plugin-template --github pulp_rpm' to update this file. 10 | # 11 | # For more info visit https://github.com/pulp/plugin_template 12 | 13 | set -mveuo pipefail 14 | 15 | # make sure this script runs at the repo root 16 | cd "$(dirname "$(realpath -e "$0")")"/../../.. 17 | 18 | pushd ../pulp-openapi-generator 19 | rm -rf "pulp_rpm-client" 20 | 21 | ./gen-client.sh "../pulp_rpm/rpm-api.json" "rpm" ruby "pulp_rpm" 22 | 23 | pushd pulp_rpm-client 24 | gem build pulp_rpm_client 25 | tar cvf "../../pulp_rpm/rpm-ruby-client.tar" "./pulp_rpm_client-"*".gem" 26 | popd 27 | popd 28 | -------------------------------------------------------------------------------- /pulp_rpm/app/migrations/0040_rpmalternatecontentsource.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 4.2.17 on 2024-12-05 18:54 2 | 3 | from django.db import migrations, models 4 | import django.db.models.deletion 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ('rpm', '0039_disttree_digest'), 11 | ] 12 | 13 | operations = [ 14 | migrations.CreateModel( 15 | name='RpmAlternateContentSource', 16 | fields=[ 17 | ('alternatecontentsource_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, related_name='rpm_rpmalternatecontentsource', serialize=False, to='core.alternatecontentsource')), 18 | ], 19 | options={ 20 | 'default_related_name': '%(app_label)s_%(model_name)s', 21 | }, 22 | bases=('core.alternatecontentsource',), 23 | ), 24 | ] 25 | -------------------------------------------------------------------------------- /.github/workflows/scripts/post_before_script.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -euv 4 | 5 | if [[ "$TEST" == "upgrade" ]]; then 6 | exit 7 | fi 8 | 9 | cmd_stdin_prefix bash -c "cat > /var/lib/pulp/scripts/sign-metadata.sh" < pulp_rpm/tests/functional/sign-metadata.sh 10 | 11 | curl -L https://github.com/pulp/pulp-fixtures/raw/master/common/GPG-KEY-fixture-signing | cmd_stdin_prefix su pulp -c "cat > /tmp/GPG-KEY-fixture-signing" 12 | curl -L https://github.com/pulp/pulp-fixtures/raw/master/common/GPG-PRIVATE-KEY-fixture-signing | cmd_stdin_prefix su pulp -c "gpg --import" 13 | echo "0C1A894EBB86AFAE218424CADDEF3019C2D4A8CF:6:" | cmd_stdin_prefix gpg --import-ownertrust 14 | cmd_prefix chmod a+x /var/lib/pulp/scripts/sign-metadata.sh 15 | 16 | cmd_prefix su pulp -c "pulpcore-manager add-signing-service sign-metadata /var/lib/pulp/scripts/sign-metadata.sh \"pulp-fixture-signing-key\"" 17 | 18 | echo "machine pulp 19 | login admin 20 | password password 21 | " > ~/.netrc 22 | -------------------------------------------------------------------------------- /pulp_rpm/app/migrations/0032_ulnremote.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 4.2.17 on 2024-12-05 18:54 2 | 3 | from django.db import migrations, models 4 | import django.db.models.deletion 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ('rpm', '0031_modulemd_static_context'), 11 | ] 12 | 13 | operations = [ 14 | migrations.CreateModel( 15 | name='UlnRemote', 16 | fields=[ 17 | ('remote_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, related_name='rpm_ulnremote', serialize=False, to='core.remote')), 18 | ('uln_server_base_url', models.CharField(max_length=512, null=True)), 19 | ], 20 | options={ 21 | 'default_related_name': '%(app_label)s_%(model_name)s', 22 | }, 23 | bases=('core.remote',), 24 | ), 25 | ] 26 | -------------------------------------------------------------------------------- /pulp_rpm/app/serializers/acs.py: -------------------------------------------------------------------------------- 1 | from gettext import gettext as _ 2 | from rest_framework import serializers 3 | 4 | from pulpcore.plugin.serializers import AlternateContentSourceSerializer 5 | from pulp_rpm.app.models import RpmAlternateContentSource 6 | 7 | 8 | class RpmAlternateContentSourceSerializer(AlternateContentSourceSerializer): 9 | """ 10 | Serializer for RPM alternate content source. 11 | """ 12 | 13 | def validate_paths(self, paths): 14 | """Validate that paths do not start with /.""" 15 | for path in paths: 16 | if path.startswith("/"): 17 | raise serializers.ValidationError(_("Path cannot start with a slash.")) 18 | if not path.endswith("/"): 19 | raise serializers.ValidationError(_("Path must end with a slash.")) 20 | return paths 21 | 22 | class Meta: 23 | fields = AlternateContentSourceSerializer.Meta.fields 24 | model = RpmAlternateContentSource 25 | -------------------------------------------------------------------------------- /pulp_rpm/app/migrations/0016_dist_tree_nofk.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.12 on 2020-07-23 11:33 2 | 3 | from django.db import migrations, models 4 | import django.db.models.deletion 5 | 6 | 7 | def unset_main_repo_fk(apps, schema_editor): 8 | """ 9 | No longer have a Foreign Key pointing to a main repo. 10 | The only way to determine that it's a main repo is to look at the packages directory path. 11 | """ 12 | Variant = apps.get_model('rpm', 'Variant') 13 | Variant.objects.filter(packages='Packages').update(repository=None) 14 | 15 | 16 | class Migration(migrations.Migration): 17 | 18 | dependencies = [ 19 | ('rpm', '0015_repo_metadata'), 20 | ] 21 | 22 | operations = [ 23 | migrations.AlterField( 24 | model_name='variant', 25 | name='repository', 26 | field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.PROTECT, related_name='+', to='core.Repository'), 27 | ), 28 | migrations.RunPython(unset_main_repo_fk) 29 | ] 30 | -------------------------------------------------------------------------------- /pulp_rpm/app/migrations/0005_optimize_sync.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 4.2.17 on 2024-12-05 18:54 2 | 3 | from django.db import migrations, models 4 | import django.db.models.deletion 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ('rpm', '0004_add_metadata_signing_service_fk'), 11 | ] 12 | 13 | operations = [ 14 | migrations.AddField( 15 | model_name='rpmrepository', 16 | name='last_sync_remote', 17 | field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='rpm_rpmrepository', to='core.remote'), 18 | ), 19 | migrations.AddField( 20 | model_name='rpmrepository', 21 | name='last_sync_repo_version', 22 | field=models.PositiveIntegerField(default=0), 23 | ), 24 | migrations.AddField( 25 | model_name='rpmrepository', 26 | name='last_sync_revision_number', 27 | field=models.CharField(max_length=20, null=True), 28 | ), 29 | ] 30 | -------------------------------------------------------------------------------- /pulp_rpm/app/migrations/0036_checksum_type.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.24 on 2021-06-10 01:27 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ('rpm', '0035_fix_auto_publish'), 10 | ] 11 | 12 | operations = [ 13 | migrations.AlterField( 14 | model_name='rpmrepository', 15 | name='metadata_checksum_type', 16 | field=models.CharField(choices=[('unknown', 'unknown'), ('md5', 'md5'), ('sha1', 'sha1'), ('sha1', 'sha1'), ('sha224', 'sha224'), ('sha256', 'sha256'), ('sha384', 'sha384'), ('sha512', 'sha512')], max_length=10, null=True), 17 | ), 18 | migrations.AlterField( 19 | model_name='rpmrepository', 20 | name='package_checksum_type', 21 | field=models.CharField(choices=[('unknown', 'unknown'), ('md5', 'md5'), ('sha1', 'sha1'), ('sha1', 'sha1'), ('sha224', 'sha224'), ('sha256', 'sha256'), ('sha384', 'sha384'), ('sha512', 'sha512')], max_length=10, null=True), 22 | ), 23 | ] 24 | -------------------------------------------------------------------------------- /pulp_rpm/app/migrations/0006_opensuse_support.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.11 on 2020-03-23 15:11 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ('rpm', '0005_optimize_sync'), 10 | ] 11 | 12 | operations = [ 13 | migrations.AddField( 14 | model_name='updatecollectionpackage', 15 | name='relogin_suggested', 16 | field=models.BooleanField(default=False), 17 | ), 18 | migrations.AddField( 19 | model_name='updatecollectionpackage', 20 | name='restart_suggested', 21 | field=models.BooleanField(default=False), 22 | ), 23 | migrations.AlterField( 24 | model_name='updatecollection', 25 | name='name', 26 | field=models.TextField(null=True), 27 | ), 28 | migrations.AlterField( 29 | model_name='updatecollection', 30 | name='shortname', 31 | field=models.TextField(null=True), 32 | ), 33 | ] 34 | -------------------------------------------------------------------------------- /pulp_rpm/app/migrations/0038_fix_sync_optimization.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.2.8 on 2021-10-08 03:37 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ('rpm', '0037_update_json_field'), 10 | ] 11 | 12 | operations = [ 13 | migrations.RemoveField( 14 | model_name='rpmrepository', 15 | name='last_sync_remote', 16 | ), 17 | migrations.RemoveField( 18 | model_name='rpmrepository', 19 | name='last_sync_repo_version', 20 | ), 21 | migrations.RemoveField( 22 | model_name='rpmrepository', 23 | name='last_sync_repomd_checksum', 24 | ), 25 | migrations.RemoveField( 26 | model_name='rpmrepository', 27 | name='last_sync_revision_number', 28 | ), 29 | migrations.AddField( 30 | model_name='rpmrepository', 31 | name='last_sync_details', 32 | field=models.JSONField(default=dict), 33 | ), 34 | ] 35 | -------------------------------------------------------------------------------- /pulp_rpm/app/migrations/0015_repo_metadata.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.13 on 2020-07-10 09:41 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | def assign_relative_path(apps, schema_editor): 7 | repo_metadata = apps.get_model('rpm', 'RepoMetadataFile') 8 | for metadata in repo_metadata.objects.all(): 9 | metadata.relative_path = metadata.contentartifact_set.first().relative_path 10 | metadata.save() 11 | 12 | 13 | class Migration(migrations.Migration): 14 | 15 | dependencies = [ 16 | ('rpm', '0014_rpmrepository_package_retention_policy'), 17 | ] 18 | 19 | operations = [ 20 | migrations.AddField( 21 | model_name='repometadatafile', 22 | name='relative_path', 23 | field=models.TextField(default=''), 24 | preserve_default=False, 25 | ), 26 | migrations.AlterUniqueTogether( 27 | name='repometadatafile', 28 | unique_together={('data_type', 'checksum', 'relative_path')}, 29 | ), 30 | migrations.RunPython(assign_relative_path) 31 | ] 32 | -------------------------------------------------------------------------------- /docs/user/index.md: -------------------------------------------------------------------------------- 1 | # Welcome to Pulp RPM! 2 | 3 | The pulp_rpm plugin extends pulpcore to support hosting RPM family content types. 4 | 5 | If you just got here, you should take our [Getting Started with RPM](site:pulp_rpm/docs/user/tutorials/create_sync_publish/) tutorial to get your first RPM repository up and running. 6 | We also recommended that you read the [Basic Concepts](site:pulp_rpm/docs/user/learn/concepts/) section before diving into the workflows and reference material. 7 | 8 | ## Features 9 | 10 | - Sync-publish workflow: 11 | * Support for RPM Packages, Advisories, Modularity, and Comps 12 | * Support for ULN servers 13 | - Versioned Repositories so every operation is a restorable snapshot 14 | - Download content on-demand when requested by clients to reduce disk space. 15 | - Upload local RPM content 16 | - Add, remove, copy, and organize RPM content into various repositories 17 | - De-duplication of all saved content 18 | - Host content either [locally or on S3](https://github.com/pulp/pulp-oci-images/issues/649) 19 | - View distributions served by pulpcore-content in a browser 20 | 21 | -------------------------------------------------------------------------------- /CHANGES/.TEMPLATE.md: -------------------------------------------------------------------------------- 1 | {# TOWNCRIER TEMPLATE #} 2 | {% for section, _ in sections.items() %} 3 | {%- set section_slug = "-" + section|replace(" ", "-")|replace("_", "-")|lower %} 4 | {%- if section %} 5 | 6 | ### {{section}} {: #{{versiondata.version}}{{section_slug}} } 7 | {% else %} 8 | {%- set section_slug = "" %} 9 | {% endif %} 10 | {% if sections[section] %} 11 | {% for category, val in definitions.items() if category in sections[section]%} 12 | 13 | #### {{ definitions[category]['name'] }} {: #{{versiondata.version}}{{section_slug}}-{{category}} } 14 | 15 | {% if definitions[category]['showcontent'] %} 16 | {% for text, values in sections[section][category].items() %} 17 | - {{ text }} 18 | {% if values %} 19 | {{ values|join(',\n ') }} 20 | {% endif %} 21 | {% endfor %} 22 | {% else %} 23 | - {{ sections[section][category]['']|join(', ') }} 24 | {% endif %} 25 | {% if sections[section][category]|length == 0 %} 26 | 27 | No significant changes. 28 | {% else %} 29 | {% endif %} 30 | {% endfor %} 31 | {% else %} 32 | 33 | No significant changes. 34 | {% endif %} 35 | {% endfor %} 36 | 37 | --- 38 | 39 | 40 | -------------------------------------------------------------------------------- /pulp_rpm/app/migrations/0030_DATA_fix_updaterecord.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.10 on 2020-02-19 14:43 2 | 3 | 4 | from django.db import models, migrations, transaction 5 | 6 | 7 | def replace_nonestring_with_null(apps, schema_editor): 8 | with transaction.atomic(): 9 | UpdateRecord = apps.get_model('rpm', 'UpdateRecord') 10 | 11 | recs = list(UpdateRecord.objects.filter(updated_date__in=('None', '')).only('updated_date')) 12 | for rec in recs: 13 | rec.updated_date = None 14 | 15 | UpdateRecord.objects.bulk_update(recs, ['updated_date']) 16 | 17 | 18 | class Migration(migrations.Migration): 19 | 20 | dependencies = [ 21 | ('rpm', '0029_rpmpublication_sqlite_metadata'), 22 | ] 23 | 24 | operations = [ 25 | migrations.AlterField( 26 | model_name='updaterecord', 27 | name='updated_date', 28 | field=models.TextField(null=True), 29 | ), 30 | # The string "None" was stored in the database in place of "" or null 31 | migrations.RunPython(replace_nonestring_with_null), 32 | ] 33 | -------------------------------------------------------------------------------- /.ci/ansible/settings.py.j2: -------------------------------------------------------------------------------- 1 | CONTENT_ORIGIN = "{{ pulp_scheme }}://pulp:{{ 443 if pulp_scheme == 'https' else 80 }}" 2 | ANSIBLE_API_HOSTNAME = "{{ pulp_scheme }}://pulp:{{ 443 if pulp_scheme == 'https' else 80 }}" 3 | ANSIBLE_CONTENT_HOSTNAME = "{{ pulp_scheme }}://pulp:{{ 443 if pulp_scheme == 'https' else 80 }}/pulp/content" 4 | PRIVATE_KEY_PATH = "/etc/pulp/certs/token_private_key.pem" 5 | PUBLIC_KEY_PATH = "/etc/pulp/certs/token_public_key.pem" 6 | TOKEN_SERVER = "{{ pulp_scheme }}://pulp:{{ 443 if pulp_scheme == 'https' else 80 }}/token/" 7 | TOKEN_SIGNATURE_ALGORITHM = "ES256" 8 | CACHE_ENABLED = True 9 | REDIS_HOST = "localhost" 10 | REDIS_PORT = 6379 11 | ANALYTICS = False 12 | 13 | {% if api_root is defined %} 14 | API_ROOT = {{ api_root | repr }} 15 | {% endif %} 16 | 17 | {% if pulp_settings %} 18 | {% for key, value in pulp_settings.items() %} 19 | {{ key | upper }} = {{ value | repr }} 20 | {% endfor %} 21 | {% endif %} 22 | 23 | {% if pulp_scenario_settings is defined and pulp_scenario_settings %} 24 | {% for key, value in pulp_scenario_settings.items() %} 25 | {{ key | upper }} = {{ value | repr }} 26 | {% endfor %} 27 | {% endif %} 28 | 29 | -------------------------------------------------------------------------------- /.github/workflows/codeql-analysis.yml: -------------------------------------------------------------------------------- 1 | # WARNING: DO NOT EDIT! 2 | # 3 | # This file was generated by plugin_template, and is managed by it. Please use 4 | # './plugin-template --github pulp_rpm' to update this file. 5 | # 6 | # For more info visit https://github.com/pulp/plugin_template 7 | name: "Rpm CodeQL" 8 | 9 | on: 10 | workflow_dispatch: 11 | schedule: 12 | - cron: '37 1 * * 6' 13 | 14 | concurrency: 15 | group: ${{ github.ref_name }}-${{ github.workflow }} 16 | cancel-in-progress: true 17 | 18 | jobs: 19 | analyze: 20 | name: Analyze 21 | runs-on: ubuntu-latest 22 | permissions: 23 | actions: read 24 | contents: read 25 | security-events: write 26 | 27 | strategy: 28 | fail-fast: false 29 | matrix: 30 | language: [ 'python' ] 31 | 32 | steps: 33 | - name: Checkout repository 34 | uses: actions/checkout@v4 35 | 36 | - name: Initialize CodeQL 37 | uses: github/codeql-action/init@v2 38 | with: 39 | languages: ${{ matrix.language }} 40 | 41 | - name: Perform CodeQL Analysis 42 | uses: github/codeql-action/analyze@v2 43 | -------------------------------------------------------------------------------- /.ci/ansible/build_container.yaml: -------------------------------------------------------------------------------- 1 | # Ansible playbook to create the pulp service containers image 2 | --- 3 | - hosts: localhost 4 | gather_facts: false 5 | vars_files: 6 | - vars/main.yaml 7 | tasks: 8 | - name: "Generate Containerfile from template" 9 | template: 10 | src: Containerfile.j2 11 | dest: Containerfile 12 | 13 | - name: "Build pulp image" 14 | # We build from the ../.. (parent dir of pulpcore git repo) Docker build 15 | # "context" so that repos like pulp-smash are accessible to Docker 16 | # build. So that PR branches can be used via relative paths. 17 | # 18 | # We default to using the docker build / podman buildah cache, for 19 | # 1-off-builds and CI purposes (which has no cache across CI runs.) 20 | # Run build.yaml with -e cache=false if your builds are using outdated 21 | # layers. 22 | command: "docker build --network host --no-cache={{ not cache | default(true) | bool }} -t {{ image.name }}:{{ image.tag }} -f {{ playbook_dir }}/Containerfile ../../.." 23 | 24 | - name: "Clean image cache" 25 | docker_prune: 26 | images : true 27 | ... 28 | -------------------------------------------------------------------------------- /pulp_rpm/app/migrations/0058_alter_addon_repository_alter_variant_repository.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 4.2.7 on 2023-11-29 23:12 2 | 3 | from django.db import migrations, models 4 | import django.db.models.deletion 5 | 6 | 7 | class Migration(migrations.Migration): 8 | dependencies = [ 9 | ("core", "0114_remove_task_args_remove_task_kwargs"), 10 | ("rpm", "0057_rpmpublication_checksum_type_and_more"), 11 | ] 12 | 13 | operations = [ 14 | migrations.AlterField( 15 | model_name="addon", 16 | name="repository", 17 | field=models.ForeignKey( 18 | on_delete=django.db.models.deletion.RESTRICT, 19 | related_name="addons", 20 | to="core.repository", 21 | ), 22 | ), 23 | migrations.AlterField( 24 | model_name="variant", 25 | name="repository", 26 | field=models.ForeignKey( 27 | null=True, 28 | on_delete=django.db.models.deletion.RESTRICT, 29 | related_name="variants", 30 | to="core.repository", 31 | ), 32 | ), 33 | ] 34 | -------------------------------------------------------------------------------- /.github/workflows/update-labels.yml: -------------------------------------------------------------------------------- 1 | # WARNING: DO NOT EDIT! 2 | # 3 | # This file was generated by plugin_template, and is managed by it. Please use 4 | # './plugin-template --github pulp_rpm' to update this file. 5 | # 6 | # For more info visit https://github.com/pulp/plugin_template 7 | 8 | 9 | --- 10 | name: "Rpm Update Labels" 11 | on: 12 | push: 13 | branches: 14 | - "main" 15 | paths: 16 | - "template_config.yml" 17 | 18 | jobs: 19 | update_backport_labels: 20 | runs-on: "ubuntu-latest" 21 | steps: 22 | - uses: "actions/setup-python@v5" 23 | with: 24 | python-version: "3.11" 25 | - name: "Configure Git with pulpbot name and email" 26 | run: | 27 | git config --global user.name 'pulpbot' 28 | git config --global user.email 'pulp-infra@redhat.com' 29 | - name: "Install python dependencies" 30 | run: | 31 | echo ::group::PYDEPS 32 | pip install requests pyyaml 33 | echo ::endgroup:: 34 | - uses: "actions/checkout@v4" 35 | - name: "Update labels" 36 | run: | 37 | python3 .github/workflows/scripts/update_backport_labels.py 38 | env: 39 | GITHUB_TOKEN: "${{ secrets.RELEASE_TOKEN }}" 40 | -------------------------------------------------------------------------------- /pulp_rpm/app/migrations/0064_remove_rpmrepository_original_checksum_types_and_more.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 4.2.10 on 2025-05-09 05:27 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ('rpm', '0063_rpmpublication_layout_rpmrepository_layout'), 10 | ] 11 | 12 | operations = [ 13 | migrations.RemoveField( 14 | model_name='rpmrepository', 15 | name='original_checksum_types', 16 | ), 17 | migrations.AlterField( 18 | model_name='rpmpublication', 19 | name='metadata_checksum_type', 20 | field=models.TextField(choices=[('unknown', 'unknown'), ('md5', 'md5'), ('sha1', 'sha1'), ('sha1', 'sha1'), ('sha224', 'sha224'), ('sha256', 'sha256'), ('sha384', 'sha384'), ('sha512', 'sha512')], null=True), 21 | ), 22 | migrations.AlterField( 23 | model_name='rpmpublication', 24 | name='package_checksum_type', 25 | field=models.TextField(choices=[('unknown', 'unknown'), ('md5', 'md5'), ('sha1', 'sha1'), ('sha1', 'sha1'), ('sha224', 'sha224'), ('sha256', 'sha256'), ('sha384', 'sha384'), ('sha512', 'sha512')], null=True), 26 | ), 27 | ] 28 | -------------------------------------------------------------------------------- /pulp_rpm/app/migrations/0048_artifacts_dependencies_fix.py: -------------------------------------------------------------------------------- 1 | from django.db import migrations 2 | import yaml 3 | 4 | 5 | def fixup_modulemd_artifacts_dependencies(apps, schema_editor): 6 | """ Set "artifacts" and "dependencies" for any saved post-3.19 back to the 3.18 format. 7 | """ 8 | Modulemd = apps.get_model("rpm", "Modulemd") 9 | 10 | modules_to_update = [] 11 | 12 | for mmd in Modulemd.objects.all().iterator(): 13 | modulemd = yaml.safe_load(mmd.snippet) 14 | if modulemd: 15 | mmd.artifacts = modulemd["data"].get("artifacts", {}).get("rpms", []) 16 | mmd.dependencies = modulemd["data"].get("dependencies", []) 17 | modules_to_update.append(mmd) 18 | if len(modules_to_update) >= 100: 19 | Modulemd.objects.bulk_update(modules_to_update, ["artifacts", "dependencies"]) 20 | modules_to_update.clear() 21 | 22 | Modulemd.objects.bulk_update(modules_to_update, ["artifacts", "dependencies"]) 23 | 24 | 25 | class Migration(migrations.Migration): 26 | 27 | dependencies = [ 28 | ('rpm', '0047_modulemd_datefield'), 29 | ] 30 | 31 | operations = [ 32 | migrations.RunPython(fixup_modulemd_artifacts_dependencies), 33 | ] 34 | -------------------------------------------------------------------------------- /pulp_rpm/app/schema/modulemd.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://json-schema.org/draft-07/schema#", 3 | "title": "Modulemd", 4 | "description": "Modulemd data", 5 | "type": "object", 6 | "properties": { 7 | "name": {"type": "string"}, 8 | "version": { 9 | "type": ["integer", "string"], 10 | "pattern": "^[0-9]+$" 11 | }, 12 | "context": {"type": ["number", "string"]}, 13 | "arch": {"type": "string"}, 14 | "summary": {"type": "string"}, 15 | "description": {"type": "string"}, 16 | "license": { 17 | "type": "object", 18 | "properties": { 19 | "module": {"type": "array"}, 20 | "content": {"type": "array"} 21 | }, 22 | "required": ["module"] 23 | } 24 | }, 25 | "required": ["name", "stream", "version", "context", "arch", "summary", "description", "license"], 26 | "additionalProperties": true, 27 | "if": { 28 | "properties": { 29 | "components": {"type": "object"} 30 | } 31 | }, 32 | "then": { 33 | "components": { 34 | "type": "object", 35 | "patternProperties": { 36 | ".": { 37 | "type": "object", 38 | "properties": { 39 | "rationale": {"type": "string"} 40 | } 41 | } 42 | } 43 | } 44 | } 45 | } -------------------------------------------------------------------------------- /pulp_rpm/app/serializers/__init__.py: -------------------------------------------------------------------------------- 1 | from .acs import ( # noqa 2 | RpmAlternateContentSourceSerializer, 3 | ) 4 | from .advisory import ( # noqa 5 | MinimalUpdateRecordSerializer, 6 | UpdateCollectionSerializer, 7 | UpdateRecordSerializer, 8 | ) 9 | from .comps import ( # noqa 10 | CompsXmlSerializer, 11 | PackageCategorySerializer, 12 | PackageEnvironmentSerializer, 13 | PackageGroupSerializer, 14 | PackageLangpacksSerializer, 15 | ) 16 | from .custom_metadata import RepoMetadataFileSerializer # noqa 17 | from .distribution import ( # noqa 18 | AddonSerializer, 19 | ChecksumSerializer, 20 | DistributionTreeSerializer, 21 | ImageSerializer, 22 | VariantSerializer, 23 | ) 24 | from .modulemd import ( # noqa 25 | ModulemdSerializer, 26 | ModulemdDefaultsSerializer, 27 | ModulemdObsoleteSerializer, 28 | ) 29 | from .package import PackageSerializer, PackageUploadSerializer, MinimalPackageSerializer # noqa 30 | from .prune import PrunePackagesSerializer # noqa 31 | from .repository import ( # noqa 32 | CopySerializer, 33 | RpmDistributionSerializer, 34 | RpmPublicationSerializer, 35 | RpmRemoteSerializer, 36 | UlnRemoteSerializer, 37 | RpmRepositorySerializer, 38 | RpmRepositorySyncURLSerializer, 39 | ) 40 | -------------------------------------------------------------------------------- /.github/workflows/docs.yml: -------------------------------------------------------------------------------- 1 | # WARNING: DO NOT EDIT! 2 | # 3 | # This file was generated by plugin_template, and is managed by it. Please use 4 | # './plugin-template --github pulp_rpm' to update this file. 5 | # 6 | # For more info visit https://github.com/pulp/plugin_template 7 | 8 | --- 9 | name: "Docs CI" 10 | on: 11 | workflow_call: 12 | inputs: 13 | run_docs: 14 | description: "Whether to run docs jobs" 15 | required: true 16 | type: string 17 | 18 | jobs: 19 | changelog: 20 | runs-on: "ubuntu-latest" 21 | defaults: 22 | run: 23 | working-directory: "pulp_rpm" 24 | steps: 25 | - uses: "actions/checkout@v4" 26 | with: 27 | fetch-depth: 1 28 | path: "pulp_rpm" 29 | - uses: "actions/setup-python@v5" 30 | with: 31 | python-version: "3.12" 32 | - name: "Install python dependencies" 33 | run: | 34 | echo ::group::PYDEPS 35 | pip install towncrier 36 | echo ::endgroup:: 37 | - name: "Build changelog" 38 | run: | 39 | towncrier build --yes --version 4.0.0.ci 40 | docs: 41 | if: ${{ inputs.run_docs == '1' }} 42 | uses: 'pulp/pulp-docs/.github/workflows/docs-ci.yml@main' 43 | with: 44 | pulpdocs_ref: 'main' 45 | -------------------------------------------------------------------------------- /pulp_rpm/app/migrations/0061_fix_modulemd_defaults_digest.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 4.2.10 on 2024-04-15 18:44 2 | 3 | import hashlib 4 | 5 | from django.db import migrations 6 | 7 | 8 | def add_snippet_hash(apps, schema_editor): 9 | """Calculate and add digest hash of the snippet.""" 10 | 11 | ModulemdDefaults = apps.get_model("rpm", "ModulemdDefaults") 12 | modules_to_update = [] 13 | for mmd in ModulemdDefaults.objects.filter(digest__in=('', None)).only("snippet").iterator(): 14 | mmd.digest = hashlib.sha256(mmd.snippet.encode()).hexdigest() 15 | modules_to_update.append(mmd) 16 | ModulemdDefaults.objects.bulk_update(modules_to_update, fields=["digest"]) 17 | 18 | Modulemd = apps.get_model("rpm", "Modulemd") 19 | modules_to_update = [] 20 | for mmd in Modulemd.objects.filter(digest__in=('', None)).only("snippet").iterator(): 21 | mmd.digest = hashlib.sha256(mmd.snippet.encode()).hexdigest() 22 | modules_to_update.append(mmd) 23 | Modulemd.objects.bulk_update(modules_to_update, fields=["digest"]) 24 | 25 | 26 | class Migration(migrations.Migration): 27 | dependencies = [ 28 | ("rpm", "0060_rpmpublication_compression_type_empty"), 29 | ] 30 | 31 | operations = [ 32 | migrations.RunPython( 33 | code=add_snippet_hash, 34 | reverse_code=migrations.RunPython.noop, 35 | ), 36 | ] 37 | -------------------------------------------------------------------------------- /pulp_rpm/app/migrations/0049_profiles_fix.py: -------------------------------------------------------------------------------- 1 | from django.db import migrations 2 | import yaml 3 | 4 | 5 | def fixup_modulemd_profiles(apps, schema_editor): 6 | """ Set "profiles" for any saved post-3.19 back to the 3.18 format. 7 | """ 8 | Modulemd = apps.get_model("rpm", "Modulemd") 9 | 10 | modules_to_update = [] 11 | 12 | for mmd in Modulemd.objects.all().iterator(): 13 | modulemd = yaml.safe_load(mmd.snippet) 14 | if modulemd: 15 | unprocessed_profiles = modulemd["data"].get("profiles", {}) 16 | profiles = {} 17 | if unprocessed_profiles: 18 | for name, data in unprocessed_profiles.items(): 19 | rpms = data.get("rpms") 20 | if rpms: 21 | profiles[name] = rpms 22 | mmd.profiles = profiles 23 | modules_to_update.append(mmd) 24 | 25 | if len(modules_to_update) >= 100: 26 | Modulemd.objects.bulk_update(modules_to_update, ["profiles"]) 27 | modules_to_update.clear() 28 | 29 | Modulemd.objects.bulk_update(modules_to_update, ["profiles"]) 30 | 31 | 32 | class Migration(migrations.Migration): 33 | 34 | dependencies = [ 35 | ('rpm', '0048_artifacts_dependencies_fix'), 36 | ] 37 | 38 | operations = [ 39 | migrations.RunPython(fixup_modulemd_profiles), 40 | ] 41 | -------------------------------------------------------------------------------- /pulp_rpm/app/migrations/0007_checksum_types.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.12 on 2020-04-08 17:09 2 | 3 | import django.contrib.postgres.fields.jsonb 4 | from django.db import migrations, models 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ('rpm', '0006_opensuse_support'), 11 | ] 12 | 13 | operations = [ 14 | migrations.AddField( 15 | model_name='rpmpublication', 16 | name='metadata_checksum_type', 17 | field=models.CharField(choices=[('unknown', 'unknown'), ('md5', 'md5'), ('sha1', 'sha1'), ('sha1', 'sha1'), ('sha224', 'sha224'), ('sha256', 'sha256'), ('sha384', 'sha384'), ('sha512', 'sha512')], default='sha256', max_length=10), 18 | preserve_default=False, 19 | ), 20 | migrations.AddField( 21 | model_name='rpmpublication', 22 | name='package_checksum_type', 23 | field=models.CharField(choices=[('unknown', 'unknown'), ('md5', 'md5'), ('sha1', 'sha1'), ('sha1', 'sha1'), ('sha224', 'sha224'), ('sha256', 'sha256'), ('sha384', 'sha384'), ('sha512', 'sha512')], default='sha256', max_length=10), 24 | preserve_default=False, 25 | ), 26 | migrations.AddField( 27 | model_name='rpmrepository', 28 | name='original_checksum_types', 29 | field=django.contrib.postgres.fields.jsonb.JSONField(default=dict), 30 | ), 31 | ] 32 | -------------------------------------------------------------------------------- /pulp_rpm/app/migrations/0039_disttree_digest.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.2.9 on 2021-12-20 22:05 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | def generate_fake_digest(apps, schema_editor): 7 | """ 8 | Generate a temporary digest for a dist tree 9 | 10 | This is needed so the dist tree is fixed by a subsequent sync. 11 | We can't use digest from .treeinfo file because a dist tree object is potentially wrong and 12 | we need to let a subsequent [force] sync to fix it. 13 | """ 14 | DistributionTree = apps.get_model('rpm', 'DistributionTree') 15 | dist_trees = DistributionTree.objects.all() 16 | for idx, dist_tree in enumerate(dist_trees): 17 | dist_tree.digest = 'temp str {0:032d}, to be removed at sync'.format(idx) 18 | DistributionTree.objects.bulk_update(dist_trees, fields=['digest'], batch_size=500) 19 | 20 | 21 | class Migration(migrations.Migration): 22 | 23 | dependencies = [ 24 | ('rpm', '0038_fix_sync_optimization'), 25 | ] 26 | 27 | operations = [ 28 | migrations.AddField( 29 | model_name='distributiontree', 30 | name='digest', 31 | field=models.CharField(default='', max_length=64), 32 | preserve_default=False, 33 | ), 34 | migrations.RunPython( 35 | generate_fake_digest 36 | ), 37 | migrations.AlterUniqueTogether( 38 | name='distributiontree', 39 | unique_together={('digest',)}, 40 | ), 41 | ] 42 | -------------------------------------------------------------------------------- /pulp_rpm/app/models/custom_metadata.py: -------------------------------------------------------------------------------- 1 | from logging import getLogger 2 | 3 | from django.db import models 4 | 5 | from pulpcore.plugin.models import Content 6 | from pulpcore.plugin.util import get_domain_pk 7 | from pulp_rpm.app.constants import CHECKSUM_CHOICES 8 | 9 | log = getLogger(__name__) 10 | 11 | 12 | class RepoMetadataFile(Content): 13 | """ 14 | Model for custom/unknown repository metadata. 15 | 16 | Fields: 17 | data_type (Text): 18 | Metadata type 19 | checksum_type (Text): 20 | Checksum type for the file 21 | checksum (Text): 22 | Checksum value for the file 23 | 24 | """ 25 | 26 | TYPE = "repo_metadata_file" 27 | UNSUPPORTED_METADATA = ["prestodelta", "deltainfo"] 28 | 29 | data_type = models.TextField() 30 | checksum_type = models.TextField(choices=CHECKSUM_CHOICES) 31 | checksum = models.TextField() 32 | relative_path = models.TextField() 33 | 34 | repo_key_fields = ("data_type",) 35 | 36 | _pulp_domain = models.ForeignKey("core.Domain", default=get_domain_pk, on_delete=models.PROTECT) 37 | 38 | class Meta: 39 | default_related_name = "%(app_label)s_%(model_name)s" 40 | unique_together = ("_pulp_domain", "data_type", "checksum", "relative_path") 41 | 42 | @property 43 | def unsupported_metadata_type(self): 44 | """ 45 | Metadata files that are known to contain deltarpm's are unsupported! 46 | """ 47 | return self.data_type in self.UNSUPPORTED_METADATA 48 | -------------------------------------------------------------------------------- /pulp_rpm/app/replica.py: -------------------------------------------------------------------------------- 1 | from pulpcore.plugin.replica import Replicator 2 | 3 | from pulp_glue.rpm.context import ( 4 | PulpRpmDistributionContext, 5 | PulpRpmPublicationContext, 6 | PulpRpmRepositoryContext, 7 | ) 8 | 9 | 10 | from pulp_rpm.app.models import RpmDistribution, RpmRemote, RpmRepository 11 | from pulp_rpm.app.tasks import synchronize as rpm_synchronize 12 | 13 | 14 | class RpmReplicator(Replicator): 15 | repository_ctx_cls = PulpRpmRepositoryContext 16 | distribution_ctx_cls = PulpRpmDistributionContext 17 | publication_ctx_cls = PulpRpmPublicationContext 18 | app_label = "rpm" 19 | remote_model_cls = RpmRemote 20 | repository_model_cls = RpmRepository 21 | distribution_model_cls = RpmDistribution 22 | distribution_serializer_name = "RpmDistributionSerializer" 23 | repository_serializer_name = "RpmRepositorySerializer" 24 | remote_serializer_name = "RpmRemoteSerializer" 25 | sync_task = rpm_synchronize 26 | 27 | def repository_extra_fields(self, remote): 28 | """Returns a dictionary where each key is a field on an RpmRemote.""" 29 | return dict(autopublish=False) 30 | 31 | def sync_params(self, repository, remote): 32 | """Returns a dictionary where key is a parameter for the sync task.""" 33 | return dict( 34 | remote_pk=remote.pk, 35 | repository_pk=repository.pk, 36 | sync_policy="mirror_complete", 37 | skip_types=[], 38 | optimize=True, 39 | ) 40 | 41 | 42 | REPLICATION_ORDER = [RpmReplicator] 43 | -------------------------------------------------------------------------------- /pulp_rpm/app/migrations/0035_fix_auto_publish.py: -------------------------------------------------------------------------------- 1 | from django.db import migrations, transaction 2 | from django.core.exceptions import ObjectDoesNotExist 3 | 4 | def remove_publications_from_auto_distributed(apps, schema_editor): 5 | with transaction.atomic(): 6 | RpmDistribution = apps.get_model("rpm", "RpmDistribution") 7 | distributions = RpmDistribution.objects.filter(repository__isnull=False, publication__isnull=False) 8 | distributions.update(publication=None) 9 | 10 | def add_publications_to_auto_distributed(apps, schema_editor): 11 | with transaction.atomic(): 12 | RpmDistribution = apps.get_model("rpm", "RpmDistribution") 13 | distributions = list(RpmDistribution.objects.filter(repository__isnull=False).select_related("repository")) 14 | for distribution in distributions: 15 | repo_version = distribution.repository.latest_version() 16 | try: 17 | publication = repo_version.publication_set.earliest("pulp_created") 18 | except ObjectDoesNotExist: 19 | publication = None 20 | distribution.publication = publication 21 | RpmDistribution.objects.bulk_update(distributions, ['publication']) 22 | 23 | 24 | class Migration(migrations.Migration): 25 | 26 | dependencies = [ 27 | ('rpm', '0034_auto_publish'), 28 | ] 29 | 30 | operations = [ 31 | migrations.RunPython( 32 | remove_publications_from_auto_distributed, 33 | reverse_code=add_publications_to_auto_distributed 34 | ) 35 | ] 36 | -------------------------------------------------------------------------------- /.flake8: -------------------------------------------------------------------------------- 1 | # WARNING: DO NOT EDIT! 2 | # 3 | # This file was generated by plugin_template, and is managed by it. Please use 4 | # './plugin-template --github pulp_rpm' to update this file. 5 | # 6 | # For more info visit https://github.com/pulp/plugin_template 7 | [flake8] 8 | exclude = ./docs/*,*/migrations/* 9 | per-file-ignores = */__init__.py: F401 10 | 11 | ignore = E203,W503,Q000,Q003,D100,D104,D106,D200,D205,D400,D401,D402,F824 12 | max-line-length = 100 13 | 14 | # Flake8 builtin codes 15 | # -------------------- 16 | # E203: no whitespace around ':'. disabled until https://github.com/PyCQA/pycodestyle/issues/373 is fixed 17 | # W503: This enforces operators before line breaks which is not pep8 or black compatible. 18 | # F824: 'nonlocal' is unused: name is never assigned in scope 19 | 20 | # Flake8-quotes extension codes 21 | # ----------------------------- 22 | # Q000: double or single quotes only, default is double (don't want to enforce this) 23 | # Q003: Change outer quotes to avoid escaping inner quotes 24 | 25 | # Flake8-docstring extension codes 26 | # -------------------------------- 27 | # D100: missing docstring in public module 28 | # D104: missing docstring in public package 29 | # D106: missing docstring in public nested class (complains about "class Meta:" and documenting those is silly) 30 | # D200: one-line docstring should fit on one line with quotes 31 | # D205: 1 blank line required between summary line and description 32 | # D400: First line should end with a period 33 | # D401: first line should be imperative (nitpicky) 34 | # D402: first line should not be the function’s “signature” (false positives) 35 | -------------------------------------------------------------------------------- /pulp_rpm/app/migrations/0041_modulemdobsolete.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.2.13 on 2022-07-15 08:42 2 | 3 | from django.db import migrations, models 4 | import django.db.models.deletion 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ('rpm', '0040_rpmalternatecontentsource'), 11 | ] 12 | 13 | operations = [ 14 | migrations.CreateModel( 15 | name='ModulemdObsolete', 16 | fields=[ 17 | ('content_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, related_name='rpm_modulemdobsolete', serialize=False, to='core.content')), 18 | ('modified', models.DateTimeField()), 19 | ('module_name', models.TextField()), 20 | ('module_stream', models.TextField()), 21 | ('message', models.TextField()), 22 | ('override_previous', models.BooleanField(null=True)), 23 | ('module_context', models.TextField(null=True)), 24 | ('eol_date', models.DateTimeField(null=True)), 25 | ('obsoleted_by_module_name', models.TextField(null=True)), 26 | ('obsoleted_by_module_stream', models.TextField(null=True)), 27 | ('snippet', models.TextField()), 28 | ], 29 | options={ 30 | 'default_related_name': '%(app_label)s_%(model_name)s', 31 | 'unique_together': {('modified', 'module_name', 'module_stream')}, 32 | }, 33 | bases=('core.content',), 34 | ), 35 | ] 36 | -------------------------------------------------------------------------------- /pulp_rpm/app/serializers/custom_metadata.py: -------------------------------------------------------------------------------- 1 | from gettext import gettext as _ 2 | from rest_framework import serializers 3 | 4 | from pulpcore.plugin.serializers import ( 5 | ContentChecksumSerializer, 6 | SingleArtifactContentUploadSerializer, 7 | ) 8 | from pulpcore.plugin.util import get_domain_pk 9 | 10 | from pulp_rpm.app.models import RepoMetadataFile 11 | 12 | 13 | class RepoMetadataFileSerializer(SingleArtifactContentUploadSerializer, ContentChecksumSerializer): 14 | """ 15 | RepoMetadataFile serializer. 16 | """ 17 | 18 | data_type = serializers.CharField(help_text=_("Metadata type.")) 19 | checksum_type = serializers.CharField(help_text=_("Checksum type for the file.")) 20 | checksum = serializers.CharField(help_text=_("Checksum for the file.")) 21 | relative_path = serializers.CharField(help_text=_("Relative path of the file.")) 22 | 23 | # Mirror unique_together from the Model 24 | def retrieve(self, validated_data): 25 | content = RepoMetadataFile.objects.filter( 26 | data_type=validated_data["data_type"], 27 | checksum=validated_data["checksum"], 28 | relative_path=validated_data["relative_path"], 29 | pulp_domain=get_domain_pk(), 30 | ) 31 | return content.first() 32 | 33 | class Meta: 34 | fields = ( 35 | ContentChecksumSerializer.Meta.fields 36 | + SingleArtifactContentUploadSerializer.Meta.fields 37 | + ( 38 | "data_type", 39 | "checksum_type", 40 | "checksum", 41 | ) 42 | ) 43 | model = RepoMetadataFile 44 | -------------------------------------------------------------------------------- /pulp_rpm/app/access_policy.py: -------------------------------------------------------------------------------- 1 | from logging import getLogger 2 | 3 | from pulpcore.plugin.models import RepositoryVersion 4 | from pulpcore.plugin.viewsets import NamedModelViewSet 5 | 6 | from pulp_rpm.app.models.repository import RpmRepository 7 | 8 | _logger = getLogger(__name__) 9 | 10 | 11 | def has_perms_to_copy(request, view, action): 12 | """ 13 | Check if the source and destination repository matches the usernames permissions. 14 | 15 | `Fail` the check at first missing permission. 16 | """ 17 | serializer = view.serializer_class(data=request.data, context={"request": request}) 18 | serializer.is_valid(raise_exception=True) 19 | 20 | for copy_action in serializer.data["config"]: 21 | dest_repo = NamedModelViewSet().get_resource(copy_action["dest_repo"], RpmRepository) 22 | 23 | # Check if user has permissions to destination repository 24 | if not ( 25 | request.user.has_perm("rpm.modify_content_rpmrepository", dest_repo) 26 | or request.user.has_perm("rpm.modify_content_rpmrepository") 27 | ): 28 | return False 29 | 30 | # Get source repo object 31 | source_version = NamedModelViewSet().get_resource( 32 | copy_action["source_repo_version"], RepositoryVersion 33 | ) 34 | 35 | source_repo = RpmRepository.objects.get(pk=source_version.repository_id) 36 | 37 | # Check if user has permissions to source repository 38 | if not ( 39 | request.user.has_perm("rpm.view_rpmrepository", source_repo) 40 | or request.user.has_perm("rpm.view_rpmrepository") 41 | ): 42 | return False 43 | 44 | return True 45 | -------------------------------------------------------------------------------- /pulp_rpm/app/migrations/0062_rpmpackagesigningservice_and_more.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 4.2.10 on 2024-04-25 16:39 2 | 3 | from django.db import migrations, models 4 | import django.db.models.deletion 5 | 6 | 7 | class Migration(migrations.Migration): 8 | dependencies = [ 9 | ("rpm", "0061_fix_modulemd_defaults_digest"), 10 | ] 11 | 12 | operations = [ 13 | migrations.CreateModel( 14 | name="RpmPackageSigningService", 15 | fields=[ 16 | ( 17 | "signingservice_ptr", 18 | models.OneToOneField( 19 | auto_created=True, 20 | on_delete=django.db.models.deletion.CASCADE, 21 | parent_link=True, 22 | primary_key=True, 23 | serialize=False, 24 | to="core.signingservice", 25 | ), 26 | ), 27 | ], 28 | options={ 29 | "abstract": False, 30 | }, 31 | bases=("core.signingservice",), 32 | ), 33 | migrations.AddField( 34 | model_name="rpmrepository", 35 | name="package_signing_fingerprint", 36 | field=models.TextField(max_length=40, null=True), 37 | ), 38 | migrations.AddField( 39 | model_name="rpmrepository", 40 | name="package_signing_service", 41 | field=models.ForeignKey( 42 | null=True, 43 | on_delete=django.db.models.deletion.SET_NULL, 44 | to="rpm.rpmpackagesigningservice", 45 | ), 46 | ), 47 | ] 48 | -------------------------------------------------------------------------------- /pulp_rpm/app/comps.py: -------------------------------------------------------------------------------- 1 | import hashlib 2 | import libcomps 3 | 4 | 5 | def list_to_idlist(lst): 6 | """ 7 | Convert list to libcomps IdList object. 8 | 9 | Args: 10 | list: a list 11 | 12 | Returns: 13 | idlist: a libcomps IdList 14 | 15 | """ 16 | idlist = libcomps.IdList() 17 | 18 | for i in lst: 19 | group_id = libcomps.GroupId(i["name"], i["default"]) 20 | if group_id not in idlist: 21 | idlist.append(group_id) 22 | 23 | return idlist 24 | 25 | 26 | def strdict_to_dict(value): 27 | """ 28 | Convert libcomps StrDict type object to standard dict. 29 | 30 | Args: 31 | value: a libcomps StrDict 32 | 33 | Returns: 34 | lang_dict: a dict 35 | 36 | """ 37 | lang_dict = {} 38 | if len(value): 39 | for i, j in value.items(): 40 | lang_dict[i] = j 41 | return lang_dict 42 | 43 | 44 | def dict_to_strdict(value): 45 | """ 46 | Convert standard dict object to libcomps StrDict type object. 47 | 48 | Args: 49 | value: a dict 50 | 51 | Returns: 52 | strdict: a libcomps StrDict 53 | 54 | """ 55 | strdict = libcomps.StrDict() 56 | for i, j in value.items(): 57 | strdict[i] = j 58 | return strdict 59 | 60 | 61 | def dict_digest(dict): 62 | """ 63 | Calculate a hexdigest for a given dictionary. 64 | 65 | Args: 66 | dict: a dictionary 67 | 68 | Returns: 69 | A digest 70 | 71 | """ 72 | prep_hash = list(dict.values()) 73 | str_prep_hash = [str(i) for i in prep_hash] 74 | str_prep_hash.sort() 75 | return hashlib.sha256("".join(str_prep_hash).encode("utf-8")).hexdigest() 76 | -------------------------------------------------------------------------------- /pulp_rpm/app/migrations/0027_checksum_null.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.16 on 2020-10-09 14:58 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | def check_checksum(apps, schema_editor): 7 | checksum_model = apps.get_model("rpm", "Checksum") 8 | repository_model = apps.get_model("rpm", "RpmRepository") 9 | q_bad_dist_tree = checksum_model.objects.filter(checksum=None) 10 | if q_bad_dist_tree: 11 | dist_tree_pks = { 12 | dt.distribution_tree.pk 13 | for dt in q_bad_dist_tree 14 | } 15 | repository_list = { 16 | f"/pulp/api/v3/repositories/rpm/rpm/{repo.pk}" 17 | for repo in repository_model.objects.filter(content__in=dist_tree_pks) 18 | } 19 | raise ValueError( 20 | "Your Pulp instance contains DistributionTree content units with null checksums. " 21 | "They were not properly validated before this update. Please remove the affected " 22 | f"DistributionTrees content from your repositories {repository_list} by removing " 23 | "old repository versions and running orphan cleanup afterwards. " 24 | "If you need to get removed content back, please re-sync corresponding repositories." 25 | ) 26 | 27 | 28 | class Migration(migrations.Migration): 29 | 30 | dependencies = [ 31 | ('rpm', '0026_add_gpgcheck_options'), 32 | ] 33 | 34 | operations = [ 35 | migrations.RunPython(check_checksum), 36 | migrations.AlterField( 37 | model_name='checksum', 38 | name='checksum', 39 | field=models.CharField(max_length=128), 40 | preserve_default=False, 41 | ), 42 | ] 43 | -------------------------------------------------------------------------------- /pulp_rpm/app/migrations/0057_rpmpublication_checksum_type_and_more.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 4.2.5 on 2023-11-07 03:51 2 | 3 | from django.db import migrations, models 4 | from django.db.models import F 5 | 6 | 7 | def set_publication_checksum(apps, schema_editor): 8 | RpmPublication = apps.get_model("rpm", "RpmPublication") 9 | RpmPublication.objects.update(checksum_type=F("metadata_checksum_type")) 10 | 11 | 12 | class Migration(migrations.Migration): 13 | 14 | dependencies = [ 15 | ('rpm', '0056_remove_rpmpublication_sqlite_metadata_and_more'), 16 | ] 17 | 18 | operations = [ 19 | migrations.AddField( 20 | model_name='rpmpublication', 21 | name='checksum_type', 22 | field=models.TextField(choices=[('unknown', 'unknown'), ('md5', 'md5'), ('sha1', 'sha1'), ('sha1', 'sha1'), ('sha224', 'sha224'), ('sha256', 'sha256'), ('sha384', 'sha384'), ('sha512', 'sha512')], null=True), 23 | ), 24 | migrations.RunPython(set_publication_checksum), 25 | migrations.AlterField( 26 | model_name='rpmpublication', 27 | name='checksum_type', 28 | field=models.TextField(choices=[('unknown', 'unknown'), ('md5', 'md5'), ('sha1', 'sha1'), ('sha1', 'sha1'), ('sha224', 'sha224'), ('sha256', 'sha256'), ('sha384', 'sha384'), ('sha512', 'sha512')]), 29 | ), 30 | migrations.AddField( 31 | model_name='rpmrepository', 32 | name='checksum_type', 33 | field=models.TextField(choices=[('unknown', 'unknown'), ('md5', 'md5'), ('sha1', 'sha1'), ('sha1', 'sha1'), ('sha224', 'sha224'), ('sha256', 'sha256'), ('sha384', 'sha384'), ('sha512', 'sha512')], null=True), 34 | ), 35 | ] 36 | -------------------------------------------------------------------------------- /pulp_rpm/app/migrations/0008_advisory_pkg_sumtype_as_int.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.11 on 2020-04-06 16:05 2 | 3 | from django.db import migrations, models, transaction 4 | 5 | 6 | def translate_sum_type(apps, schema_editor): 7 | # use sum_type as int as createrepo_c uses 8 | with transaction.atomic(): 9 | UpdateCollectionPackage = apps.get_model('rpm', 'UpdateCollectionPackage') 10 | update_collection_package_to_save = [] 11 | for package in UpdateCollectionPackage.objects.all(): 12 | package.sum_type_temp = int(package.sum_type) if package.sum_type else None 13 | update_collection_package_to_save.append(package) 14 | UpdateCollectionPackage.objects.bulk_update( 15 | update_collection_package_to_save, 16 | ['sum_type_temp'] 17 | ) 18 | 19 | 20 | class Migration(migrations.Migration): 21 | 22 | dependencies = [ 23 | ('rpm', '0007_checksum_types'), 24 | ] 25 | 26 | operations = [ 27 | migrations.AddField( 28 | model_name='updatecollectionpackage', 29 | name='sum_type_temp', 30 | field=models.PositiveIntegerField( 31 | null=True, default=None, 32 | choices=[ 33 | (0, 0), (1, 1), (2, 2), (3, 3), (4, 4), (5, 5), (6, 6), (7, 7) 34 | ] 35 | ) 36 | ), 37 | migrations.RunPython(translate_sum_type), 38 | migrations.RemoveField( 39 | model_name='updatecollectionpackage', 40 | name='sum_type' 41 | ), 42 | migrations.RenameField( 43 | model_name='updatecollectionpackage', 44 | old_name='sum_type_temp', 45 | new_name='sum_type' 46 | ) 47 | ] 48 | -------------------------------------------------------------------------------- /.github/workflows/scripts/build_python_client.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # This script expects all -api.json files to exist in the plugins root directory. 4 | # It produces a -python-client.tar and -python-client-docs.tar file in the plugins root directory. 5 | 6 | # WARNING: DO NOT EDIT! 7 | # 8 | # This file was generated by plugin_template, and is managed by it. Please use 9 | # './plugin-template --github pulp_rpm' to update this file. 10 | # 11 | # For more info visit https://github.com/pulp/plugin_template 12 | 13 | set -mveuo pipefail 14 | 15 | # make sure this script runs at the repo root 16 | cd "$(dirname "$(realpath -e "$0")")"/../../.. 17 | 18 | pushd ../pulp-openapi-generator 19 | rm -rf "pulp_rpm-client" 20 | 21 | ./gen-client.sh "../pulp_rpm/rpm-api.json" "rpm" python "pulp_rpm" 22 | 23 | pushd pulp_rpm-client 24 | python -m build 25 | 26 | twine check "dist/pulp_rpm_client-"*"-py3-none-any.whl" 27 | twine check "dist/pulp_rpm_client-"*".tar.gz" 28 | 29 | tar cvf "../../pulp_rpm/rpm-python-client.tar" ./dist 30 | 31 | find ./docs/* -exec sed -i 's/Back to README/Back to HOME/g' {} \; 32 | find ./docs/* -exec sed -i 's/README//g' {} \; 33 | cp README.md docs/index.md 34 | sed -i 's/docs\///g' docs/index.md 35 | find ./docs/* -exec sed -i 's/\.md//g' {} \; 36 | 37 | cat >> mkdocs.yml << DOCSYAML 38 | --- 39 | site_name: PulpRpm Client 40 | site_description: Rpm bindings 41 | site_author: Pulp Team 42 | site_url: https://docs.pulpproject.org/pulp_rpm_client/ 43 | repo_name: pulp/pulp_rpm 44 | repo_url: https://github.com/pulp/pulp_rpm 45 | theme: readthedocs 46 | DOCSYAML 47 | 48 | # Building the bindings docs 49 | mkdocs build 50 | 51 | # Pack the built site. 52 | tar cvf ../../pulp_rpm/rpm-python-client-docs.tar ./site 53 | popd 54 | popd 55 | -------------------------------------------------------------------------------- /pulp_rpm/app/exceptions.py: -------------------------------------------------------------------------------- 1 | from gettext import gettext as _ 2 | 3 | from pulpcore.plugin.exceptions import PulpException 4 | 5 | 6 | class AdvisoryConflict(PulpException): 7 | """ 8 | Raised when two advisories conflict in a way that Pulp can't resolve it. 9 | """ 10 | 11 | def __init__(self, msg): 12 | """ 13 | Set the exception identifier. 14 | 15 | Args: 16 | msg(str): Detailed message about the reasons for Advisory conflict 17 | """ 18 | super().__init__("RPM0001") 19 | self.msg = msg 20 | 21 | def __str__(self): 22 | """ 23 | Return a message for the exception. 24 | """ 25 | return self.msg 26 | 27 | 28 | class DistributionTreeConflict(FileNotFoundError): 29 | """ 30 | Raised when two or more distribution trees are being added to a repository version. 31 | """ 32 | 33 | def __init__(self, msg): 34 | """ 35 | Set the exception identifier and msg. 36 | """ 37 | super().__init__("RPM0002") 38 | self.msg = _("More than one distribution tree cannot be added to a " "repository version.") 39 | 40 | def __str__(self): 41 | """ 42 | Return a message for the exception. 43 | """ 44 | return self.msg 45 | 46 | 47 | class UlnCredentialsError(PulpException): 48 | """ 49 | Raised when no valid ULN Credentials were given. 50 | """ 51 | 52 | def __init__(self, msg): 53 | """ 54 | Set the exception identifier and msg. 55 | """ 56 | super().__init__("RPM0003") 57 | self.msg = _("No valid ULN credentials given.") 58 | 59 | def __str__(self): 60 | """ 61 | Return a message for the exception. 62 | """ 63 | return self.msg 64 | -------------------------------------------------------------------------------- /releasing.md: -------------------------------------------------------------------------------- 1 | [//]: # "WARNING: DO NOT EDIT!" 2 | [//]: # "" 3 | [//]: # "This file was generated by plugin_template, and is managed by it. Please use" 4 | [//]: # "'./plugin-template --github pulp_rpm' to update this file." 5 | [//]: # "" 6 | [//]: # "For more info visit https://github.com/pulp/plugin_template" 7 | # Releasing (For Internal Use) 8 | 9 | This document outlines the steps to perform a release. 10 | 11 | ### Determine if a Release is Required 12 | - Make sure to have GitPython python package installed 13 | - Run the release checker script: 14 | ``` 15 | python3 .ci/scripts/check_release.py 16 | ``` 17 | 18 | ### Release a New Y-version (e.g., 3.23.0) 19 | - If a new minor version (Y) is needed, trigger a [Create New Release Branch](https://github.com/pulp/pulp_rpm/actions/workflows/create-branch.yml) job from the main branch via the GitHub Actions. 20 | - Look for the "Bump minor version" pull request and merge it. 21 | - Trigger a [Release Pipeline](https://github.com/pulp/pulp_rpm/actions/workflows/release.yml) job by specifying the new release branch (X.**Y**) via the GitHub Actions. 22 | 23 | ### Release a New Z-version (Patch Release) (e.g., 3.23.1, 3.22.12) 24 | - Trigger a [Release Pipeline](https://github.com/pulp/pulp_rpm/actions/workflows/release.yml) job by specifying the release branch (X.Y) via the GitHub Actions. 25 | 26 | ## Final Steps 27 | - Ensure the new version appears on PyPI (it should appear after [Publish Release](https://github.com/pulp/pulp_rpm/actions/workflows/publish.yml) workflow succeeds). 28 | - Verify that the changelog has been updated by looking for the "Update Changelog" pull request (A new PR should be available on the next day). 29 | - [optional] Post a brief announcement about the new release on the [Pulp Discourse](https://discourse.pulpproject.org/). 30 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | # WARNING: DO NOT EDIT! 2 | # 3 | # This file was generated by plugin_template, and is managed by it. Please use 4 | # './plugin-template --github pulp_rpm' to update this file. 5 | # 6 | # For more info visit https://github.com/pulp/plugin_template 7 | 8 | --- 9 | name: Rpm Release Pipeline 10 | on: 11 | workflow_dispatch: 12 | 13 | defaults: 14 | run: 15 | working-directory: "pulp_rpm" 16 | 17 | jobs: 18 | build-artifacts: 19 | runs-on: "ubuntu-latest" 20 | 21 | strategy: 22 | fail-fast: false 23 | 24 | steps: 25 | - uses: "actions/checkout@v4" 26 | with: 27 | fetch-depth: 0 28 | path: "pulp_rpm" 29 | token: ${{ secrets.RELEASE_TOKEN }} 30 | 31 | - uses: "actions/setup-python@v5" 32 | with: 33 | python-version: "3.11" 34 | 35 | - name: "Install python dependencies" 36 | run: | 37 | echo ::group::PYDEPS 38 | pip install bump-my-version towncrier 39 | echo ::endgroup:: 40 | 41 | - name: "Configure Git with pulpbot name and email" 42 | run: | 43 | git config --global user.name 'pulpbot' 44 | git config --global user.email 'pulp-infra@redhat.com' 45 | 46 | - name: "Setting secrets" 47 | run: | 48 | python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" 49 | env: 50 | SECRETS_CONTEXT: "${{ toJson(secrets) }}" 51 | 52 | - name: "Tag the release" 53 | run: | 54 | .github/workflows/scripts/release.sh 55 | shell: "bash" 56 | env: 57 | PY_COLORS: "1" 58 | ANSIBLE_FORCE_COLOR: "1" 59 | GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" 60 | GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" 61 | -------------------------------------------------------------------------------- /pulp_rpm/app/migrations/0034_auto_publish.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.19 on 2021-04-02 21:07 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ('rpm', '0033_new_distribution_model'), 10 | ] 11 | 12 | operations = [ 13 | migrations.AddField( 14 | model_name='rpmrepository', 15 | name='autopublish', 16 | field=models.BooleanField(default=False), 17 | ), 18 | migrations.AddField( 19 | model_name='rpmrepository', 20 | name='gpgcheck', 21 | field=models.IntegerField(choices=[(0, 0), (1, 1)], default=0), 22 | ), 23 | migrations.AddField( 24 | model_name='rpmrepository', 25 | name='metadata_checksum_type', 26 | field=models.CharField(choices=[('unknown', 'unknown'), ('md5', 'md5'), ('sha1', 'sha1'), ('sha1', 'sha1'), ('sha224', 'sha224'), ('sha256', 'sha256'), ('sha384', 'sha384'), ('sha512', 'sha512')], default='sha256', max_length=10), 27 | ), 28 | migrations.AddField( 29 | model_name='rpmrepository', 30 | name='package_checksum_type', 31 | field=models.CharField(choices=[('unknown', 'unknown'), ('md5', 'md5'), ('sha1', 'sha1'), ('sha1', 'sha1'), ('sha224', 'sha224'), ('sha256', 'sha256'), ('sha384', 'sha384'), ('sha512', 'sha512')], default='sha256', max_length=10), 32 | ), 33 | migrations.AddField( 34 | model_name='rpmrepository', 35 | name='repo_gpgcheck', 36 | field=models.IntegerField(choices=[(0, 0), (1, 1)], default=0), 37 | ), 38 | migrations.AddField( 39 | model_name='rpmrepository', 40 | name='sqlite_metadata', 41 | field=models.BooleanField(default=False), 42 | ), 43 | ] 44 | -------------------------------------------------------------------------------- /.github/workflows/scripts/before_script.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # WARNING: DO NOT EDIT! 4 | # 5 | # This file was generated by plugin_template, and is managed by it. Please use 6 | # './plugin-template --github pulp_rpm' to update this file. 7 | # 8 | # For more info visit https://github.com/pulp/plugin_template 9 | 10 | # make sure this script runs at the repo root 11 | cd "$(dirname "$(realpath -e "$0")")"/../../.. 12 | 13 | set -euv 14 | 15 | source .github/workflows/scripts/utils.sh 16 | 17 | export PRE_BEFORE_SCRIPT=$PWD/.github/workflows/scripts/pre_before_script.sh 18 | export POST_BEFORE_SCRIPT=$PWD/.github/workflows/scripts/post_before_script.sh 19 | 20 | if [[ -f $PRE_BEFORE_SCRIPT ]]; then 21 | source $PRE_BEFORE_SCRIPT 22 | fi 23 | 24 | # Developers should be able to reproduce the containers with this config 25 | echo "CI vars:" 26 | tail -v -n +1 .ci/ansible/vars/main.yaml 27 | 28 | # Developers often want to know the final pulp config 29 | echo "PULP CONFIG:" 30 | tail -v -n +1 .ci/ansible/settings/settings.* ~/.config/pulp_smash/settings.json 31 | 32 | echo "Containerfile:" 33 | tail -v -n +1 .ci/ansible/Containerfile 34 | 35 | echo "Constraints Files:" 36 | # The need not even exist. 37 | tail -v -n +1 ../*/*constraints.txt || true 38 | 39 | # Needed for some functional tests 40 | cmd_prefix bash -c "echo '%wheel ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/nopasswd" 41 | cmd_prefix bash -c "usermod -a -G wheel pulp" 42 | 43 | if [[ "${REDIS_DISABLED:-false}" == true ]]; then 44 | cmd_prefix bash -c "s6-rc -d change redis" 45 | echo "The Redis service was disabled for $TEST" 46 | fi 47 | 48 | if [[ -f $POST_BEFORE_SCRIPT ]]; then 49 | source $POST_BEFORE_SCRIPT 50 | fi 51 | 52 | # Lots of plugins try to use this path, and throw warnings if they cannot access it. 53 | cmd_prefix mkdir /.pytest_cache 54 | cmd_prefix chown pulp:pulp /.pytest_cache 55 | -------------------------------------------------------------------------------- /pulp_rpm/app/migrations/0052_modulemd_digest.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 4.2.4 on 2023-09-04 14:06 2 | 3 | from django.db import migrations, models 4 | import hashlib 5 | 6 | 7 | def add_snippet_hash(apps, schema_editor): 8 | """Calculate and add digest hash of the snippet.""" 9 | 10 | Modulemd = apps.get_model("rpm", "Modulemd") 11 | modules_to_update = [] 12 | 13 | for mmd in Modulemd.objects.all().only("snippet").iterator(): 14 | # for the modules that have empty string snippet will be calculated digest of 15 | # e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 16 | mmd.digest = hashlib.sha256(mmd.snippet.encode()).hexdigest() 17 | modules_to_update.append(mmd) 18 | Modulemd.objects.bulk_update(modules_to_update, fields=["digest"]) 19 | 20 | 21 | class Migration(migrations.Migration): 22 | 23 | dependencies = [ 24 | ("rpm", "0051_alter_distributiontree_unique_together_and_more"), 25 | ] 26 | 27 | operations = [ 28 | migrations.AddField( 29 | model_name="modulemd", 30 | name="digest", 31 | field=models.TextField(null=True), 32 | ), 33 | migrations.RunPython( 34 | code=add_snippet_hash, 35 | reverse_code=migrations.RunPython.noop, 36 | elidable=True, 37 | ), 38 | migrations.AlterField( 39 | model_name="modulemd", 40 | name="digest", 41 | field=models.TextField(), 42 | ), 43 | migrations.AlterUniqueTogether( 44 | name="modulemd", 45 | unique_together=set(), 46 | ), 47 | migrations.AlterUniqueTogether( 48 | name="modulemd", 49 | unique_together={ 50 | ("_pulp_domain", "name", "stream", "version", "context", "arch", "digest") 51 | }, 52 | ), 53 | ] 54 | -------------------------------------------------------------------------------- /docs/user/guides/sign-packages.md: -------------------------------------------------------------------------------- 1 | # Sign RPM Packages 2 | 3 | Sign an RPM Package using a registered RPM signing service. 4 | 5 | Currently, only on-upload signing is supported. 6 | 7 | ## On Upload 8 | 9 | !!! tip "New in 3.26.0 (Tech Preview)" 10 | 11 | Sign an RPM Package when uploading it to a Repository. 12 | 13 | ### Pre-requisites 14 | 15 | - Have an `RpmPackageSigningService` registered 16 | (see [here](site:pulp_rpm/docs/admin/guides/add-signing-services/#package-signing)). 17 | - Have the V4 fingerprint of the key you want to use. The key should be accessible by the SigningService you are using. 18 | 19 | ### Instructions 20 | 21 | 1. Configure a Repository to enable signing. 22 | - Both `package_signing_service` and `package_signing_fingerprint` must be set. 23 | - If they are set, any package upload to the Repository will be signed by the service. 24 | 2. Upload a Package to this Repository. 25 | 26 | ### Example 27 | 28 | ```bash 29 | # Create a Repository w/ required params 30 | http POST $API_ROOT/repositories/rpm/rpm \ 31 | name="MyRepo" \ 32 | package_signing_service=$SIGNING_SERVICE_HREF \ 33 | package_signing_fingerprint=$SIGNING_FINGERPRINT 34 | 35 | # Upload a package 36 | pulp rpm content upload \ 37 | --repository ${REPOSITORY} \ 38 | --file ${FILE} 39 | ``` 40 | 41 | ### Known Limitations 42 | 43 | **Traffic overhead**: The signing of a package should happen inside of a Pulp worker. 44 | [By design](site:pulpcore/docs/dev/learn/plugin-concepts/#tasks), 45 | Pulp needs to temporarily commit the file to the default backend storage in order to make the Uploaded File available to the tasking system. 46 | This implies in some extra traffic, compared to a scenario where a task could process the file directly. 47 | 48 | **No sign tracking**: We do not track signing information of a package. 49 | 50 | For extra context, see discussion [here](https://github.com/pulp/pulp_rpm/issues/2986). 51 | -------------------------------------------------------------------------------- /docs/user/guides/alternate-content-source.md: -------------------------------------------------------------------------------- 1 | # Configure Alternate Content Sources 2 | 3 | Alternate Content Sources (ACS) can help speed up populating of new repositories. 4 | If you have content stored locally or geographically near you which matches 5 | the remote content, Alternate Content Sources will allow you to substitute 6 | this content, allowing for faster data transfer. 7 | 8 | *Alternate Content Sources* base is provided by pulpcore. 9 | You can learn more about its general usage [here](site:pulp_file/docs/admin/guides/alternate-content-sources/). 10 | 11 | To use an Alternate Content Source you need a `RPMRemote` with path of your ACS. 12 | 13 | !!! warning 14 | Remotes with mirrorlist URLs cannot be used as an Alternative Content Source. 15 | 16 | 17 | ```bash 18 | pulp rpm remote create --name rpm_acs_remote --policy on_demand --url http://fixtures.pulpproject.org/rpm-unsigned/ 19 | ``` 20 | 21 | ## Create Alternate Content Source 22 | 23 | Create an Alternate Content Source. 24 | 25 | ```bash 26 | pulp rpm acs create --name rpm_acs --remote rpm_acs_remote 27 | ``` 28 | 29 | ### Alternate Content Source Paths 30 | 31 | If you have more places with ACS within one base path you can specify them 32 | by paths and all of them will be considered as a ACS. 33 | 34 | ```bash 35 | pulp rpm remote create --name rpm_acs_remote --policy on_demand --url http://fixtures.pulpproject.org/ 36 | pulp rpm acs create --name rpm_acs --remote rpm_acs_remote --path "rpm-unsigned/" --path "rpm-distribution-tree/" 37 | ``` 38 | 39 | ## Refresh Alternate Content Source 40 | 41 | To make your ACS available for future syncs you need to call `refresh` endpoint 42 | on your ACS. This create a catalogue of available content which will be used instead 43 | new content if found. 44 | 45 | ```bash 46 | pulp rpm acs refresh --name rpm_acs 47 | ``` 48 | 49 | Alternate Content Source has a global scope so if any content is found in ACS it 50 | will be used in all future syncs. 51 | -------------------------------------------------------------------------------- /pulp_rpm/tests/functional/api/test_advisory_conflict.py: -------------------------------------------------------------------------------- 1 | """Tests to test advisory conflict resolution functionality.""" 2 | 3 | from pulp_rpm.tests.functional.constants import ( 4 | RPM_ADVISORY_DIFFERENT_PKGLIST_URL, 5 | RPM_ADVISORY_TEST_ID, 6 | RPM_UNSIGNED_FIXTURE_URL, 7 | ) 8 | 9 | 10 | def test_two_advisories_same_id_to_repo( 11 | rpm_repository_api, 12 | rpm_advisory_api, 13 | rpm_repository_factory, 14 | init_and_sync, 15 | monitor_task, 16 | delete_orphans_pre, 17 | ): 18 | """ 19 | Test when two different advisories with the same id are added to a repo. 20 | 21 | Should merge the two advisories into a single one. 22 | """ 23 | # Setup 24 | repo_rpm_unsigned, _ = init_and_sync(url=RPM_UNSIGNED_FIXTURE_URL) 25 | repo_rpm_advisory_diffpkgs, _ = init_and_sync(url=RPM_ADVISORY_DIFFERENT_PKGLIST_URL) 26 | advisory_rpm_unsigned_href = ( 27 | rpm_advisory_api.list( 28 | repository_version=repo_rpm_unsigned.latest_version_href, 29 | id=RPM_ADVISORY_TEST_ID, 30 | ) 31 | .results[0] 32 | .pulp_href 33 | ) 34 | advisory_rpm_advisory_diffpkgs_href = ( 35 | rpm_advisory_api.list( 36 | repository_version=repo_rpm_advisory_diffpkgs.latest_version_href, 37 | id=RPM_ADVISORY_TEST_ID, 38 | ) 39 | .results[0] 40 | .pulp_href 41 | ) 42 | 43 | # Test advisory conflicts 44 | repo = rpm_repository_factory() 45 | 46 | data = { 47 | "add_content_units": [ 48 | advisory_rpm_unsigned_href, 49 | advisory_rpm_advisory_diffpkgs_href, 50 | ] 51 | } 52 | response = rpm_repository_api.modify(repo.pulp_href, data) 53 | monitor_task(response.task) 54 | a_repo = rpm_repository_api.read(repo.pulp_href) 55 | 56 | duplicated_advisory_list = rpm_advisory_api.list( 57 | repository_version=a_repo.latest_version_href, 58 | id=RPM_ADVISORY_TEST_ID, 59 | ).results 60 | assert 1 == len(duplicated_advisory_list) 61 | -------------------------------------------------------------------------------- /.ci/ansible/Containerfile.j2: -------------------------------------------------------------------------------- 1 | FROM {{ ci_base | default(pulp_default_container) }} 2 | 3 | # Add source directories to container 4 | {% for item in plugins %} 5 | ADD ./{{ item.name }} ./{{ item.name }} 6 | {% endfor %} 7 | 8 | {% for item in extra_files | default([]) %} 9 | ADD ./{{ item.origin }} {{ item.destination }} 10 | {% endfor %} 11 | 12 | # This MUST be the ONLY call to pip install in inside the container. 13 | RUN pip3 install --upgrade pip setuptools wheel && \ 14 | rm -rf /root/.cache/pip && \ 15 | pip3 install 16 | {%- if s3_test | default(false) -%} 17 | {{ " " }}git+https://github.com/gerrod3/botocore.git@fix-100-continue 18 | {%- endif -%} 19 | {%- for item in plugins -%} 20 | {{ " " }}{{ item.source }} 21 | {%- if item.upperbounds | default(false) -%} 22 | {{ " " }}-c ./{{ item.name }}/upperbounds_constraints.txt 23 | {%- endif -%} 24 | {%- if item.lowerbounds | default(false) -%} 25 | {{ " " }}-c ./{{ item.name }}/lowerbounds_constraints.txt 26 | {%- endif -%} 27 | {%- if item.ci_requirements | default(false) -%} 28 | {{ " " }}-r ./{{ item.name }}/ci_requirements.txt 29 | {%- endif -%} 30 | {%- endfor %} 31 | {{ " " }}-c ./{{ plugins[0].name }}/.ci/assets/ci_constraints.txt && \ 32 | rm -rf /root/.cache/pip 33 | 34 | {% if pulp_env is defined and pulp_env %} 35 | {% for key, value in pulp_env.items() %} 36 | ENV {{ key | upper }}={{ value }} 37 | {% endfor %} 38 | {% endif %} 39 | 40 | {% if pulp_scenario_env is defined and pulp_scenario_env %} 41 | {% for key, value in pulp_scenario_env.items() %} 42 | ENV {{ key | upper }}={{ value }} 43 | {% endfor %} 44 | {% endif %} 45 | 46 | USER pulp:pulp 47 | RUN PULP_STATIC_ROOT=/var/lib/operator/static/ PULP_CONTENT_ORIGIN=localhost \ 48 | /usr/local/bin/pulpcore-manager collectstatic --clear --noinput --link 49 | USER root:root 50 | 51 | {% for item in plugins %} 52 | RUN export plugin_path="$(pip3 show {{ item.name }} | sed -n -e 's/Location: //p')/{{ item.name }}" && \ 53 | ln $plugin_path/app/webserver_snippets/nginx.conf /etc/nginx/pulp/{{ item.name }}.conf || true 54 | {% endfor %} 55 | 56 | ENTRYPOINT ["/init"] 57 | -------------------------------------------------------------------------------- /pulp_rpm/tests/functional/api/test_character_encoding.py: -------------------------------------------------------------------------------- 1 | """Tests for Pulp's characters encoding.""" 2 | 3 | import uuid 4 | 5 | import pytest 6 | import requests 7 | 8 | from pulpcore.tests.functional.utils import PulpTaskError 9 | from pulp_rpm.tests.functional.constants import ( 10 | RPM_WITH_NON_ASCII_NAME, 11 | RPM_WITH_NON_ASCII_URL, 12 | RPM_WITH_NON_UTF_8_NAME, 13 | RPM_WITH_NON_UTF_8_URL, 14 | ) 15 | 16 | 17 | """Test upload of RPMs with different character encoding. 18 | 19 | This test targets the following issues: 20 | 21 | * `Pulp #4210 `_ 22 | * `Pulp #4215 `_ 23 | """ 24 | 25 | 26 | def test_upload_non_ascii( 27 | tmp_path, pulpcore_bindings, rpm_package_api, monitor_task, delete_orphans_pre 28 | ): 29 | """Test whether one can upload an RPM with non-ascii metadata.""" 30 | temp_file = tmp_path / str(uuid.uuid4()) 31 | temp_file.write_bytes(requests.get(RPM_WITH_NON_ASCII_URL).content) 32 | artifact = pulpcore_bindings.ArtifactsApi.create(str(temp_file)) 33 | response = rpm_package_api.create( 34 | artifact=artifact.pulp_href, 35 | relative_path=RPM_WITH_NON_ASCII_NAME, 36 | ) 37 | task = monitor_task(response.task) 38 | assert len(task.created_resources) == 1 39 | 40 | 41 | def test_upload_non_utf8( 42 | tmp_path, pulpcore_bindings, rpm_package_api, monitor_task, delete_orphans_pre 43 | ): 44 | """Test whether an exception is raised when non-utf-8 is uploaded.""" 45 | temp_file = tmp_path / str(uuid.uuid4()) 46 | temp_file.write_bytes(requests.get(RPM_WITH_NON_UTF_8_URL).content) 47 | artifact = pulpcore_bindings.ArtifactsApi.create(str(temp_file)) 48 | with pytest.raises(PulpTaskError) as ctx: 49 | response = rpm_package_api.create( 50 | artifact=artifact.pulp_href, 51 | relative_path=RPM_WITH_NON_UTF_8_NAME, 52 | ) 53 | monitor_task(response.task) 54 | 55 | error_msg = ctx.value.task.error["description"] 56 | assert "'utf-8' codec can't decode byte 0x80 in position 168: invalid start" in error_msg 57 | -------------------------------------------------------------------------------- /.ci/scripts/pr_labels.py: -------------------------------------------------------------------------------- 1 | #!/bin/env python3 2 | 3 | # This script is running with elevated privileges from the main branch against pull requests. 4 | 5 | import re 6 | import sys 7 | import tomllib 8 | from pathlib import Path 9 | 10 | from git import Repo 11 | 12 | 13 | def main(): 14 | assert len(sys.argv) == 3 15 | 16 | with open("pyproject.toml", "rb") as fp: 17 | PYPROJECT_TOML = tomllib.load(fp) 18 | BLOCKING_REGEX = re.compile(r"DRAFT|WIP|NO\s*MERGE|DO\s*NOT\s*MERGE|EXPERIMENT") 19 | ISSUE_REGEX = re.compile(r"(?:fixes|closes)[\s:]+#(\d+)") 20 | CHERRY_PICK_REGEX = re.compile(r"^\s*\(cherry picked from commit [0-9a-f]*\)\s*$") 21 | try: 22 | CHANGELOG_EXTS = { 23 | f".{item['directory']}" for item in PYPROJECT_TOML["tool"]["towncrier"]["type"] 24 | } 25 | except KeyError: 26 | CHANGELOG_EXTS = {".feature", ".bugfix", ".doc", ".removal", ".misc"} 27 | 28 | repo = Repo(".") 29 | 30 | base_commit = repo.commit(sys.argv[1]) 31 | head_commit = repo.commit(sys.argv[2]) 32 | 33 | pr_commits = list(repo.iter_commits(f"{base_commit}..{head_commit}")) 34 | 35 | labels = { 36 | "multi-commit": len(pr_commits) > 1, 37 | "cherry-pick": False, 38 | "no-issue": False, 39 | "no-changelog": False, 40 | "wip": False, 41 | } 42 | for commit in pr_commits: 43 | labels["wip"] |= BLOCKING_REGEX.search(commit.summary) is not None 44 | no_issue = ISSUE_REGEX.search(commit.message, re.IGNORECASE) is None 45 | labels["no-issue"] |= no_issue 46 | cherry_pick = CHERRY_PICK_REGEX.search(commit.message) is not None 47 | labels["cherry-pick"] |= cherry_pick 48 | changelog_snippets = [ 49 | k 50 | for k in commit.stats.files 51 | if k.startswith("CHANGES/") and Path(k).suffix in CHANGELOG_EXTS 52 | ] 53 | labels["no-changelog"] |= not changelog_snippets 54 | 55 | print("ADD_LABELS=" + ",".join((k for k, v in labels.items() if v))) 56 | print("REMOVE_LABELS=" + ",".join((k for k, v in labels.items() if not v))) 57 | 58 | 59 | if __name__ == "__main__": 60 | main() 61 | -------------------------------------------------------------------------------- /pulp_rpm/app/migrations/0017_merge_advisory_collections.py: -------------------------------------------------------------------------------- 1 | from django.db import ( 2 | migrations, 3 | models, 4 | ) 5 | 6 | 7 | def do_clone(apps, advisory, in_collection): 8 | # We're cloning collection - find its updatecollectionpackages 9 | uc_packages = in_collection.packages.all() 10 | # break the advisory/collection link 11 | in_collection.update_record.remove(advisory) 12 | # create a new copy of the collection and link it to the advisory 13 | new_collection = in_collection 14 | new_collection.pk = None 15 | new_collection.save() 16 | # need to have an id before we can build the m2m relation 17 | new_collection.update_record.add(advisory) 18 | new_collection.save() 19 | new_packages = [] 20 | for a_package in uc_packages.iterator(): 21 | # create copies of the package list and link to new collection 22 | a_package.pk = None 23 | a_package.update_collection = new_collection 24 | new_packages.append(a_package) 25 | UpdateCollectionPackage = apps.get_model("rpm", "UpdateCollectionPackage") 26 | UpdateCollectionPackage.objects.bulk_create(new_packages) 27 | 28 | 29 | def clone_reused_update_collections(apps, schema): 30 | # Find UpdateCollections that point to multiple UpdateRecords 31 | # For all but the first one, create a clone 32 | UpdateCollection = apps.get_model("rpm", "UpdateCollection") 33 | collections = UpdateCollection.objects.annotate( 34 | num_advisories=models.Count('update_record')).filter( 35 | num_advisories__gte=2).all().iterator() 36 | for collection in collections: 37 | # Look at all the advisories this collection is associated with 38 | advisories = collection.update_record.all() 39 | # Skip the first adviroy found; for any that remain, disconnect and clone 40 | for advisory in advisories[1:]: 41 | do_clone(apps, advisory, collection) 42 | 43 | class Migration(migrations.Migration): 44 | 45 | dependencies = [ 46 | ('rpm', '0016_dist_tree_nofk'), 47 | ] 48 | 49 | operations = [ 50 | migrations.RunPython(clone_reused_update_collections), 51 | ] 52 | -------------------------------------------------------------------------------- /.github/workflows/scripts/update_backport_labels.py: -------------------------------------------------------------------------------- 1 | # WARNING: DO NOT EDIT! 2 | # 3 | # This file was generated by plugin_template, and is managed by it. Please use 4 | # './plugin-template --github pulp_rpm' to update this file. 5 | # 6 | # For more info visit https://github.com/pulp/plugin_template 7 | 8 | import requests 9 | import yaml 10 | import random 11 | import os 12 | 13 | 14 | def random_color(): 15 | """Generates a random 24-bit number in hex""" 16 | color = random.randrange(0, 2**24) 17 | return format(color, "06x") 18 | 19 | 20 | session = requests.Session() 21 | token = os.getenv("GITHUB_TOKEN") 22 | 23 | headers = { 24 | "Authorization": f"token {token}", 25 | "Accept": "application/vnd.github+json", 26 | "X-GitHub-Api-Version": "2022-11-28", 27 | } 28 | session.headers.update(headers) 29 | 30 | # get all labels from the repository's current state 31 | response = session.get("https://api.github.com/repos/pulp/pulp_rpm/labels", headers=headers) 32 | assert response.status_code == 200 33 | old_labels = set([x["name"] for x in response.json() if x["name"].startswith("backport-")]) 34 | 35 | # get list of branches from template_config.yml 36 | with open("./template_config.yml", "r") as f: 37 | plugin_template = yaml.safe_load(f) 38 | branches = set(plugin_template["supported_release_branches"]) 39 | latest_release_branch = plugin_template["latest_release_branch"] 40 | if latest_release_branch is not None: 41 | branches.add(latest_release_branch) 42 | new_labels = {"backport-" + x for x in branches} 43 | 44 | # delete old labels that are not in new labels 45 | for label in old_labels.difference(new_labels): 46 | response = session.delete( 47 | f"https://api.github.com/repos/pulp/pulp_rpm/labels/{label}", headers=headers 48 | ) 49 | assert response.status_code == 204 50 | 51 | # create new labels that are not in old labels 52 | for label in new_labels.difference(old_labels): 53 | color = random_color() 54 | response = session.post( 55 | "https://api.github.com/repos/pulp/pulp_rpm/labels", 56 | headers=headers, 57 | json={"name": label, "color": color}, 58 | ) 59 | assert response.status_code == 201 60 | -------------------------------------------------------------------------------- /.github/workflows/lint.yml: -------------------------------------------------------------------------------- 1 | # WARNING: DO NOT EDIT! 2 | # 3 | # This file was generated by plugin_template, and is managed by it. Please use 4 | # './plugin-template --github pulp_rpm' to update this file. 5 | # 6 | # For more info visit https://github.com/pulp/plugin_template 7 | 8 | --- 9 | name: "Lint" 10 | on: 11 | workflow_call: 12 | 13 | defaults: 14 | run: 15 | working-directory: "pulp_rpm" 16 | 17 | jobs: 18 | lint: 19 | runs-on: "ubuntu-latest" 20 | 21 | steps: 22 | - uses: "actions/checkout@v4" 23 | with: 24 | fetch-depth: 1 25 | path: "pulp_rpm" 26 | 27 | - uses: "actions/setup-python@v5" 28 | with: 29 | python-version: "3.11" 30 | 31 | - name: "Install python dependencies" 32 | run: | 33 | echo ::group::PYDEPS 34 | pip install -r lint_requirements.txt 35 | echo ::endgroup:: 36 | 37 | - name: "Lint workflow files" 38 | run: | 39 | yamllint -s -d '{extends: relaxed, rules: {line-length: disable}}' .github/workflows 40 | 41 | - name: "Verify bump version config" 42 | run: | 43 | bump-my-version bump --dry-run release 44 | bump-my-version show-bump 45 | 46 | # run black separately from flake8 to get a diff 47 | - name: "Run black" 48 | run: | 49 | black --version 50 | black --check --diff . 51 | 52 | # Lint code. 53 | - name: "Run flake8" 54 | run: | 55 | flake8 56 | 57 | - name: "Run extra lint checks" 58 | run: | 59 | [ ! -x .ci/scripts/extra_linting.sh ] || .ci/scripts/extra_linting.sh 60 | 61 | - name: "Check for any files unintentionally left out of MANIFEST.in" 62 | run: | 63 | check-manifest 64 | 65 | - name: "Verify requirements files" 66 | run: | 67 | python .ci/scripts/check_requirements.py 68 | 69 | - name: "Check for pulpcore imports outside of pulpcore.plugin" 70 | run: | 71 | sh .ci/scripts/check_pulpcore_imports.sh 72 | 73 | - name: "Check for common gettext problems" 74 | run: | 75 | sh .ci/scripts/check_gettext.sh 76 | -------------------------------------------------------------------------------- /pulp_rpm/app/migrations/0003_DATA_incorrect_json.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.10 on 2020-02-19 14:43 2 | 3 | 4 | import json 5 | import yaml 6 | 7 | from django.db import migrations, transaction 8 | import django.contrib.postgres.fields.jsonb 9 | 10 | 11 | def unflatten_json(apps, schema_editor): 12 | # re: https://pulp.plan.io/issues/6191 13 | with transaction.atomic(): 14 | Modulemd = apps.get_model("rpm", "Modulemd") 15 | for module in Modulemd.objects.all().only("artifacts", "dependencies", "_artifacts"): 16 | a = module._artifacts.first() 17 | snippet = a.file.read().decode() 18 | module_dict = yaml.safe_load(snippet) 19 | a.file.close() 20 | module.artifacts = json.loads(module.artifacts) 21 | module.dependencies = module_dict["data"]["dependencies"] 22 | module.save() 23 | 24 | ModulemdDefaults = apps.get_model("rpm", "ModulemdDefaults") 25 | for mod_defs in ModulemdDefaults.objects.all().only("profiles"): 26 | mod_defs.profiles = json.loads(mod_defs.profiles) 27 | mod_defs.save() 28 | 29 | 30 | def replace_emptystring_with_null(apps, schema_editor): 31 | # An empty string can't be converted to JSON, so use a null instead 32 | with transaction.atomic(): 33 | UpdateCollection = apps.get_model("rpm", "UpdateCollection") 34 | for coll in UpdateCollection.objects.all().only("module"): 35 | if coll.module == "": 36 | coll.module = "null" # de-serializes to None, 37 | coll.save() 38 | 39 | 40 | class Migration(migrations.Migration): 41 | 42 | dependencies = [ 43 | ("rpm", "0002_updaterecord_reboot_suggested"), 44 | ] 45 | 46 | operations = [ 47 | # Bugfixes 48 | migrations.RunPython(unflatten_json), 49 | # In-place migrate JSON stored in text field to a JSONField 50 | migrations.RunPython(replace_emptystring_with_null), 51 | migrations.AlterField( 52 | model_name="updatecollection", 53 | name="module", 54 | field=django.contrib.postgres.fields.jsonb.JSONField(null=True), 55 | ), 56 | ] 57 | -------------------------------------------------------------------------------- /pulp_rpm/app/migrations/0046_rbac_perms.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.2.13 on 2022-04-21 15:14 2 | 3 | from django.db import migrations 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ('rpm', '0045_modulemd_fields'), 10 | ] 11 | 12 | operations = [ 13 | migrations.AlterModelOptions( 14 | name='rpmalternatecontentsource', 15 | options={'default_related_name': '%(app_label)s_%(model_name)s', 'permissions': [('refresh_rpmalternatecontentsource', 'Refresh an Alternate Content Source'), ('manage_roles_rpmalternatecontentsource', 'Can manage roles on ACS')]}, 16 | ), 17 | migrations.AlterModelOptions( 18 | name='rpmdistribution', 19 | options={'default_related_name': '%(app_label)s_%(model_name)s', 'permissions': [('manage_roles_rpmdistribution', 'Can manage roles on an RPM distribution')]}, 20 | ), 21 | migrations.AlterModelOptions( 22 | name='rpmpublication', 23 | options={'default_related_name': '%(app_label)s_%(model_name)s', 'permissions': [('manage_roles_rpmpublication', 'Can manage roles on an RPM publication')]}, 24 | ), 25 | migrations.AlterModelOptions( 26 | name='rpmremote', 27 | options={'default_related_name': '%(app_label)s_%(model_name)s', 'permissions': [('manage_roles_rpmremote', 'Can manage roles on an RPM remotes')]}, 28 | ), 29 | migrations.AlterModelOptions( 30 | name='rpmrepository', 31 | options={'default_related_name': '%(app_label)s_%(model_name)s', 'permissions': [('manage_roles_rpmrepository', 'Can manage roles on RPM repositories'), ('modify_content_rpmrepository', 'Add content to, or remove content from a repository'), ('repair_rpmrepository', 'Copy a repository'), ('sync_rpmrepository', 'Sync a repository'), ('delete_rpmrepository_version', 'Delete a repository version')]}, 32 | ), 33 | migrations.AlterModelOptions( 34 | name='ulnremote', 35 | options={'default_related_name': '%(app_label)s_%(model_name)s', 'permissions': [('manage_roles_ulnremote', 'Can manage roles on an ULN remotes')]}, 36 | ), 37 | ] 38 | -------------------------------------------------------------------------------- /.github/workflows/scripts/pre_before_install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Simple check if a test was added and 'coverage.md' was updated with PR. 3 | 4 | set -euv 5 | 6 | # skip this check for everything but PRs 7 | if [ "$GITHUB_EVENT_NAME" != "pull_request" ]; then 8 | return 0 9 | fi 10 | 11 | COMMIT_BEFORE=$(jq --raw-output .before "$GITHUB_EVENT_PATH") 12 | COMMIT_AFTER=$(jq --raw-output .after "$GITHUB_EVENT_PATH") 13 | 14 | 15 | RANGE=`echo ${COMMIT_BEFORE}..${COMMIT_AFTER}` 16 | COMMIT_RANGE=`echo ${COMMIT_BEFORE}...${COMMIT_AFTER}` 17 | 18 | # check for code changes 19 | if [[ ! `git log --no-merges --pretty='format:' --name-only "$RANGE" | grep -v "pulp_rpm/__init__.py" | grep "pulp_rpm/.*.py" || true` ]] 20 | then 21 | echo "No code changes detected. Skipping coverage and test check." 22 | return 0 23 | fi 24 | 25 | # check if a test was added 26 | NEEDS_TEST="$(git diff --name-only $COMMIT_RANGE | grep -E 'feature|bugfix' || true)" 27 | CONTAINS_TEST="$(git diff --name-only $COMMIT_RANGE | grep -E 'test_' || true)" 28 | 29 | if [[ $(git log --format=medium --no-merges "$RANGE" | grep "\[notest\]" || true) ]] 30 | then 31 | echo "[notest] is present - skipping the check for the test requirement" 32 | elif [ -n "$NEEDS_TEST" ] && [ -z "$CONTAINS_TEST" ]; then 33 | echo "Every feature and bugfix should come with a test." 34 | exit 1 35 | fi 36 | 37 | if [[ $(git log --format=medium --no-merges "$RANGE" | grep "\[nocoverage\]" || true) ]] 38 | then 39 | echo "[nocoverage] is present - skipping this check" 40 | return 0 41 | fi 42 | 43 | coverage_file_name="coverage.md" 44 | coverage_original_file_name="coverage_original.md" 45 | master_version="https://raw.githubusercontent.com/pulp/pulp_rpm/master/${coverage_file_name}" 46 | 47 | # get original file from master 48 | curl --silent $master_version -o $coverage_original_file_name 49 | 50 | # check if coverage.md was updated and clean 51 | if diff -qs $coverage_file_name $coverage_original_file_name 52 | then 53 | echo "ERROR: coverage.md file is not updated." 54 | echo "Please update 'coverage.md' file if you are adding a new feature or updating an existing one." 55 | rm $coverage_original_file_name 56 | exit 1 57 | else 58 | rm $coverage_original_file_name 59 | return 0 60 | fi 61 | -------------------------------------------------------------------------------- /pulp_rpm/app/migrations/0047_modulemd_datefield.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.2.16 on 2022-11-08 19:12 2 | 3 | from django.db import migrations, models 4 | 5 | import yaml 6 | 7 | 8 | def parse_date(apps, schema_editor): 9 | """Parse dates from snippet.""" 10 | ModulemdObsolete = apps.get_model("rpm", "ModulemdObsolete") 11 | 12 | obsoletes_to_update = [] 13 | 14 | for obsolete in ModulemdObsolete.objects.all(): 15 | parsed_snippet = yaml.safe_load(obsolete.snippet) 16 | obsolete.modified = parsed_snippet['data']['modified'] 17 | if parsed_snippet['data'].get('eol_date'): 18 | obsolete.eol_date = parsed_snippet['data'].get('eol_date') 19 | obsoletes_to_update.append(obsolete) 20 | 21 | ModulemdObsolete.objects.bulk_update(obsoletes_to_update, ["modified", "eol_date"]) 22 | 23 | 24 | def clean_date_fields(apps, schema_editor): 25 | """Emtpy date fields.""" 26 | ModulemdObsolete = apps.get_model("rpm", "ModulemdObsolete") 27 | obsoletes_to_update = [] 28 | 29 | for obsolete in ModulemdObsolete.objects.all(): 30 | obsolete.modified = None 31 | obsolete.eol_date = None 32 | 33 | ModulemdObsolete.objects.bulk_update(obsoletes_to_update, ["modified", "eol_date"]) 34 | 35 | 36 | class Migration(migrations.Migration): 37 | 38 | dependencies = [ 39 | ('rpm', '0046_rbac_perms'), 40 | ] 41 | 42 | operations = [ 43 | migrations.AlterField( 44 | model_name='modulemdobsolete', 45 | name='modified', 46 | field=models.DateTimeField(null=True), 47 | ), 48 | migrations.AlterField( 49 | model_name='modulemdobsolete', 50 | name='eol_date', 51 | field=models.DateTimeField(null=True), 52 | ), 53 | migrations.RunPython(clean_date_fields), 54 | migrations.AlterField( 55 | model_name='modulemdobsolete', 56 | name='eol_date', 57 | field=models.TextField(null=True), 58 | ), 59 | migrations.AlterField( 60 | model_name='modulemdobsolete', 61 | name='modified', 62 | field=models.TextField(), 63 | ), 64 | migrations.RunPython(parse_date), 65 | ] 66 | -------------------------------------------------------------------------------- /.github/workflows/scripts/stage-changelog-for-default-branch.py: -------------------------------------------------------------------------------- 1 | # WARNING: DO NOT EDIT! 2 | # 3 | # This file was generated by plugin_template, and is managed by it. Please use 4 | # './plugin-template --github pulp_rpm' to update this file. 5 | # 6 | # For more info visit https://github.com/pulp/plugin_template 7 | 8 | import argparse 9 | import os 10 | import textwrap 11 | 12 | from git import Repo 13 | from git.exc import GitCommandError 14 | 15 | 16 | helper = textwrap.dedent( 17 | """\ 18 | Stage the changelog for a release on main branch. 19 | 20 | Example: 21 | $ python .github/workflows/scripts/stage-changelog-for-default-branch.py 3.4.0 22 | 23 | """ 24 | ) 25 | 26 | parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter, description=helper) 27 | 28 | parser.add_argument( 29 | "release_version", 30 | type=str, 31 | help="The version string for the release.", 32 | ) 33 | 34 | args = parser.parse_args() 35 | 36 | release_version_arg = args.release_version 37 | 38 | release_path = os.path.dirname(os.path.abspath(__file__)) 39 | plugin_path = release_path.split("/.github")[0] 40 | 41 | if not release_version_arg.endswith(".0"): 42 | os._exit(os.system("python .ci/scripts/changelog.py")) 43 | 44 | print(f"\n\nRepo path: {plugin_path}") 45 | repo = Repo(plugin_path) 46 | 47 | changelog_commit = None 48 | # Look for a commit with the requested release version 49 | for commit in repo.iter_commits(): 50 | if f"{release_version_arg} changelog" == commit.message.split("\n")[0]: 51 | changelog_commit = commit 52 | break 53 | if f"Add changelog for {release_version_arg}" == commit.message.split("\n")[0]: 54 | changelog_commit = commit 55 | break 56 | 57 | if not changelog_commit: 58 | raise RuntimeError("Changelog commit for {release_version_arg} was not found.") 59 | 60 | git = repo.git 61 | git.stash() 62 | git.checkout("origin/main") 63 | try: 64 | git.cherry_pick(changelog_commit.hexsha) 65 | except GitCommandError: 66 | git.add("CHANGES/") 67 | # Don't try opening an editor for the commit message 68 | with git.custom_environment(GIT_EDITOR="true"): 69 | git.cherry_pick("--continue") 70 | git.reset("origin/main") 71 | -------------------------------------------------------------------------------- /COMMITMENT: -------------------------------------------------------------------------------- 1 | GPL Cooperation Commitment, version 1.0 2 | 3 | Before filing or continuing to prosecute any legal proceeding or claim 4 | (other than a Defensive Action) arising from termination of a Covered 5 | License, we commit to extend to the person or entity ('you') accused 6 | of violating the Covered License the following provisions regarding 7 | cure and reinstatement, taken from GPL version 3. As used here, the 8 | term 'this License' refers to the specific Covered License being 9 | enforced. 10 | 11 | However, if you cease all violation of this License, then your 12 | license from a particular copyright holder is reinstated (a) 13 | provisionally, unless and until the copyright holder explicitly 14 | and finally terminates your license, and (b) permanently, if the 15 | copyright holder fails to notify you of the violation by some 16 | reasonable means prior to 60 days after the cessation. 17 | 18 | Moreover, your license from a particular copyright holder is 19 | reinstated permanently if the copyright holder notifies you of the 20 | violation by some reasonable means, this is the first time you 21 | have received notice of violation of this License (for any work) 22 | from that copyright holder, and you cure the violation prior to 30 23 | days after your receipt of the notice. 24 | 25 | We intend this Commitment to be irrevocable, and binding and 26 | enforceable against us and assignees of or successors to our 27 | copyrights. 28 | 29 | Definitions 30 | 31 | 'Covered License' means the GNU General Public License, version 2 32 | (GPLv2), the GNU Lesser General Public License, version 2.1 33 | (LGPLv2.1), or the GNU Library General Public License, version 2 34 | (LGPLv2), all as published by the Free Software Foundation. 35 | 36 | 'Defensive Action' means a legal proceeding or claim that We bring 37 | against you in response to a prior proceeding or claim initiated by 38 | you or your affiliate. 39 | 40 | 'We' means each contributor to this repository as of the date of 41 | inclusion of this file, including subsidiaries of a corporate 42 | contributor. 43 | 44 | This work is available under a [Creative Commons Attribution-ShareAlike 4.0 International license] 45 | (https://creativecommons.org/licenses/by-sa/4.0/). 46 | -------------------------------------------------------------------------------- /.github/workflows/nightly.yml: -------------------------------------------------------------------------------- 1 | # WARNING: DO NOT EDIT! 2 | # 3 | # This file was generated by plugin_template, and is managed by it. Please use 4 | # './plugin-template --github pulp_rpm' to update this file. 5 | # 6 | # For more info visit https://github.com/pulp/plugin_template 7 | 8 | --- 9 | name: "Rpm Nightly CI" 10 | on: 11 | schedule: 12 | # * is a special character in YAML so you have to quote this string 13 | # runs at 3:00 UTC daily 14 | - cron: '00 3 * * *' 15 | workflow_dispatch: 16 | 17 | defaults: 18 | run: 19 | working-directory: "pulp_rpm" 20 | 21 | concurrency: 22 | group: "${{ github.ref_name }}-${{ github.workflow }}" 23 | cancel-in-progress: true 24 | 25 | jobs: 26 | build: 27 | uses: "./.github/workflows/build.yml" 28 | 29 | test: 30 | needs: "build" 31 | uses: "./.github/workflows/test.yml" 32 | with: 33 | matrix_env: | 34 | [{"TEST": "pulp"}, {"TEST": "azure"}, {"TEST": "s3"}, {"TEST": "lowerbounds"}, {"PERFORMANCE_TEST": "sync", "TEST": "performance"}, {"PERFORMANCE_TEST": "publish", "TEST": "performance"}, {"PERFORMANCE_TEST": "pulp_to_pulp", "TEST": "performance"}] 35 | 36 | changelog: 37 | runs-on: ubuntu-latest 38 | steps: 39 | - uses: "actions/checkout@v4" 40 | with: 41 | fetch-depth: 0 42 | path: "pulp_rpm" 43 | 44 | - uses: "actions/setup-python@v5" 45 | with: 46 | python-version: "3.13" 47 | 48 | - name: "Install python dependencies" 49 | run: | 50 | echo ::group::PYDEPS 51 | pip install gitpython packaging toml 52 | echo ::endgroup:: 53 | 54 | - name: "Configure Git with pulpbot name and email" 55 | run: | 56 | git config --global user.name 'pulpbot' 57 | git config --global user.email 'pulp-infra@redhat.com' 58 | 59 | - name: Collect changes from all branches 60 | run: python .ci/scripts/collect_changes.py 61 | 62 | - name: Create Pull Request 63 | uses: peter-evans/create-pull-request@v6 64 | with: 65 | token: ${{ secrets.RELEASE_TOKEN }} 66 | title: "Update Changelog" 67 | body: "" 68 | branch: "changelog/update" 69 | delete-branch: true 70 | path: "pulp_rpm" 71 | ... 72 | -------------------------------------------------------------------------------- /pulp_rpm/tests/functional/api/test_download_policies.py: -------------------------------------------------------------------------------- 1 | """Tests for Pulp`s download policies.""" 2 | 3 | import pytest 4 | 5 | from pulp_rpm.tests.functional.constants import ( 6 | RPM_FIXTURE_SUMMARY, 7 | DOWNLOAD_POLICIES, 8 | ) 9 | from pulpcore.client.pulp_rpm import RpmRpmPublication 10 | 11 | 12 | @pytest.mark.parametrize("download_policy", DOWNLOAD_POLICIES) 13 | def test_download_policies( 14 | download_policy, 15 | init_and_sync, 16 | rpm_repository_version_api, 17 | rpm_publication_api, 18 | gen_object_with_cleanup, 19 | delete_orphans_pre, 20 | get_content_summary, 21 | ): 22 | """Sync repositories with the different ``download_policy``. 23 | 24 | Do the following: 25 | 26 | 1. Create a repository, and a remote. 27 | 2. Sync the remote. 28 | 3. Assert that repository version is not None. 29 | 4. Assert that the correct number of possible units to be downloaded 30 | were shown. 31 | 5. Sync again with the same remote. 32 | 6. Assert that the latest repository version did not change. 33 | 7. Assert that the same number of units are shown, and after the 34 | second sync no extra units should be shown, since the same remote 35 | was synced again. 36 | 8. Publish repository synced with lazy ``download_policy``. 37 | """ 38 | # Step 1, 2 39 | repo, remote = init_and_sync(policy=download_policy) 40 | 41 | # Step 3, 4 42 | assert repo.latest_version_href.endswith("/1/") 43 | content_summary = get_content_summary(repo) 44 | assert content_summary["present"] == RPM_FIXTURE_SUMMARY 45 | assert content_summary["added"] == RPM_FIXTURE_SUMMARY 46 | 47 | # Step 5 48 | latest_version_href = repo.latest_version_href 49 | repo, remote = init_and_sync(repository=repo, remote=remote) 50 | 51 | # Step 6, 7 52 | assert latest_version_href == repo.latest_version_href 53 | content_summary = get_content_summary(repo) 54 | assert content_summary["present"] == RPM_FIXTURE_SUMMARY 55 | 56 | # Step 8 57 | publish_data = RpmRpmPublication(repository=repo.pulp_href) 58 | publication = gen_object_with_cleanup(rpm_publication_api, publish_data) 59 | 60 | assert publication.repository is not None 61 | assert publication.repository_version is not None 62 | -------------------------------------------------------------------------------- /pulp_rpm/tests/functional/api/test_pulp_to_pulp.py: -------------------------------------------------------------------------------- 1 | """Tests that verify download of content served by Pulp.""" 2 | 3 | import pytest 4 | 5 | from pulp_rpm.tests.functional.constants import DOWNLOAD_POLICIES 6 | from pulpcore.client.pulp_rpm import RpmRpmPublication 7 | 8 | 9 | @pytest.mark.parallel 10 | @pytest.mark.parametrize("policy", DOWNLOAD_POLICIES) 11 | def test_pulp_pulp_sync( 12 | policy, 13 | init_and_sync, 14 | distribution_base_url, 15 | rpm_repository_version_api, 16 | rpm_publication_api, 17 | rpm_distribution_factory, 18 | gen_object_with_cleanup, 19 | ): 20 | """Verify whether content served by Pulp can be synced. 21 | 22 | The initial sync to Pulp is one of many different download policies, the second sync is 23 | immediate in order to exercise downloading all of the files. 24 | 25 | Do the following: 26 | 27 | 1. Create, populate, publish, and distribute a repository. 28 | 2. Sync other repository using as remote url, 29 | the distribution base_url from the previous repository. 30 | 31 | """ 32 | repo, remote = init_and_sync(policy=policy) 33 | 34 | # Create a publication. 35 | publish_data = RpmRpmPublication( 36 | repository=repo.pulp_href, 37 | checksum_type="sha512", 38 | ) 39 | publication = gen_object_with_cleanup(rpm_publication_api, publish_data) 40 | 41 | # Create a distribution. 42 | distribution = rpm_distribution_factory(publication=publication.pulp_href) 43 | 44 | # Create another repo pointing to distribution base_url 45 | # Should this second policy always be "immediate"? 46 | repo2, remote2 = init_and_sync( 47 | url=distribution_base_url(distribution.base_url), policy="immediate" 48 | ) 49 | repo_ver = rpm_repository_version_api.read(repo.latest_version_href) 50 | summary = {k: v["count"] for k, v in repo_ver.content_summary.present.items()} 51 | repo_ver2 = rpm_repository_version_api.read(repo2.latest_version_href) 52 | summary2 = {k: v["count"] for k, v in repo_ver2.content_summary.present.items()} 53 | assert summary == summary2 54 | 55 | added = {k: v["count"] for k, v in repo_ver.content_summary.added.items()} 56 | added2 = {k: v["count"] for k, v in repo_ver2.content_summary.added.items()} 57 | assert added == added2 58 | -------------------------------------------------------------------------------- /CONTRIBUTING.rst: -------------------------------------------------------------------------------- 1 | Contributing 2 | ============ 3 | 4 | To contribute to the ``pulp_rpm`` package follow this process: 5 | 6 | 1. Clone the GitHub repo 7 | 2. Make a change 8 | 3. Add a functional test when you fix a bug or introduce a feature 9 | 4. Make sure all tests passed 10 | 5. Add a file into CHANGES folder (Changelog update). 11 | 6. If your PR introduces a new feature or updates the existing one, update coverage.md 12 | 7. Commit changes to your own ``pulp_rpm`` clone 13 | 8. Make pull request from github page for your clone against master branch 14 | 15 | 16 | .. _changelog-update: 17 | 18 | Changelog update 19 | **************** 20 | 21 | The CHANGES.rst file is managed using the `towncrier tool `_ 22 | and all non trivial changes must be accompanied by a news entry. 23 | 24 | To add an entry to the news file, you first need an issue on github describing the change you 25 | want to make. Once you have an issue, take its number and create a file inside of the ``CHANGES/`` 26 | directory named after that issue number with an extension of .feature, .bugfix, .doc, .removal, or 27 | .misc. So if your issue is 3543 and it fixes a bug, you would create the file 28 | ``CHANGES/3543.bugfix``. 29 | 30 | PRs can span multiple categories by creating multiple files (for instance, if you added a feature 31 | and deprecated an old feature at the same time, you would create CHANGES/NNNN.feature and 32 | CHANGES/NNNN.removal). Likewise if a PR touches multiple issues/PRs you may create a file for each 33 | of them with the exact same contents and Towncrier will deduplicate them. 34 | 35 | The contents of this file are reStructuredText formatted text that will be used as the content of 36 | the news file entry. You do not need to reference the issue or PR numbers here as towncrier will 37 | automatically add a reference to all of the affected issues when rendering the news file. 38 | 39 | 40 | .. _coverage-update: 41 | 42 | Coverage update 43 | *************** 44 | 45 | The coverage.md file contains a table that keeps track of how well features are backed by functional 46 | tests. If you are adding a new feature or updating an existing one, specify in this table whether 47 | it has any coverage, partial of full coverage. 48 | Every update to this table should be accompanied with a functional test. 49 | -------------------------------------------------------------------------------- /pulp_rpm/app/migrations/0044_noartifact_modules.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.2.15 on 2022-08-12 16:28 2 | 3 | from django.core.files.storage import default_storage 4 | from django.db import migrations, models, transaction 5 | 6 | 7 | def convert_artifact_to_snippets(apps, schema_editor): 8 | """Create snippet from artifact and remove artifact.""" 9 | with transaction.atomic(): 10 | Modulemd = apps.get_model("rpm", "Modulemd") 11 | ModulemdDefaults = apps.get_model("rpm", "ModulemdDefaults") 12 | ContentArtifact = apps.get_model("core", "ContentArtifact") 13 | modules_with_snippet = [] 14 | defaults_with_snippet = [] 15 | 16 | for module in Modulemd.objects.all(): 17 | artifact = module._artifacts.get() 18 | if default_storage.exists(artifact.file.name): 19 | module.snippet = artifact.file.read().decode("utf-8") 20 | content_artifact = ContentArtifact.objects.filter(content__pk=module.pk) 21 | content_artifact.delete() 22 | modules_with_snippet.append(module) 23 | 24 | for default in ModulemdDefaults.objects.all(): 25 | artifact = default._artifacts.get() 26 | if default_storage.exists(artifact.file.name): 27 | default.snippet = artifact.file.read().decode("utf-8") 28 | content_artifact = ContentArtifact.objects.filter(content__pk=default.pk) 29 | content_artifact.delete() 30 | defaults_with_snippet.append(default) 31 | 32 | Modulemd.objects.bulk_update(modules_with_snippet, ["snippet"]) 33 | ModulemdDefaults.objects.bulk_update(defaults_with_snippet, ["snippet"]) 34 | 35 | 36 | class Migration(migrations.Migration): 37 | 38 | dependencies = [ 39 | ("rpm", "0043_textfield_conversion"), 40 | ] 41 | 42 | operations = [ 43 | migrations.AddField( 44 | model_name="modulemd", 45 | name="snippet", 46 | field=models.TextField(default=""), 47 | preserve_default=False, 48 | ), 49 | migrations.AddField( 50 | model_name="modulemddefaults", 51 | name="snippet", 52 | field=models.TextField(default=""), 53 | preserve_default=False, 54 | ), 55 | migrations.RunPython(convert_artifact_to_snippets), 56 | ] 57 | -------------------------------------------------------------------------------- /.github/workflows/scripts/before_install.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # WARNING: DO NOT EDIT! 4 | # 5 | # This file was generated by plugin_template, and is managed by it. Please use 6 | # './plugin-template --github pulp_rpm' to update this file. 7 | # 8 | # For more info visit https://github.com/pulp/plugin_template 9 | 10 | # make sure this script runs at the repo root 11 | cd "$(dirname "$(realpath -e "$0")")"/../../.. 12 | 13 | set -mveuo pipefail 14 | 15 | if [ "${GITHUB_REF##refs/heads/}" = "${GITHUB_REF}" ] 16 | then 17 | BRANCH_BUILD=0 18 | else 19 | BRANCH_BUILD=1 20 | BRANCH="${GITHUB_REF##refs/heads/}" 21 | fi 22 | if [ "${GITHUB_REF##refs/tags/}" = "${GITHUB_REF}" ] 23 | then 24 | TAG_BUILD=0 25 | else 26 | TAG_BUILD=1 27 | BRANCH="${GITHUB_REF##refs/tags/}" 28 | fi 29 | 30 | COMMIT_MSG=$(git log --format=%B --no-merges -1) 31 | export COMMIT_MSG 32 | 33 | COMPONENT_VERSION=$(python3 -c "import tomllib; print(tomllib.load(open('pyproject.toml', 'rb'))['project']['version'])") 34 | 35 | mkdir .ci/ansible/vars || true 36 | echo "---" > .ci/ansible/vars/main.yaml 37 | echo "legacy_component_name: pulp_rpm" >> .ci/ansible/vars/main.yaml 38 | echo "component_name: rpm" >> .ci/ansible/vars/main.yaml 39 | echo "component_version: '${COMPONENT_VERSION}'" >> .ci/ansible/vars/main.yaml 40 | 41 | export PRE_BEFORE_INSTALL=$PWD/.github/workflows/scripts/pre_before_install.sh 42 | export POST_BEFORE_INSTALL=$PWD/.github/workflows/scripts/post_before_install.sh 43 | 44 | if [ -f $PRE_BEFORE_INSTALL ]; then 45 | source $PRE_BEFORE_INSTALL 46 | fi 47 | 48 | if [ "$GITHUB_EVENT_NAME" = "pull_request" ] || [ "${BRANCH_BUILD}" = "1" -a "${BRANCH}" != "main" ] 49 | then 50 | echo $COMMIT_MSG | sed -n -e 's/.*CI Base Image:\s*\([-_/[:alnum:]]*:[-_[:alnum:]]*\).*/ci_base: "\1"/p' >> .ci/ansible/vars/main.yaml 51 | fi 52 | 53 | for i in {1..3} 54 | do 55 | ansible-galaxy collection install "amazon.aws:8.1.0" && s=0 && break || s=$? && sleep 3 56 | done 57 | if [[ $s -gt 0 ]] 58 | then 59 | echo "Failed to install amazon.aws" 60 | exit $s 61 | fi 62 | 63 | if [[ "$TEST" = "pulp" ]]; then 64 | python3 .ci/scripts/calc_constraints.py -u pyproject.toml > upperbounds_constraints.txt 65 | fi 66 | if [[ "$TEST" = "lowerbounds" ]]; then 67 | python3 .ci/scripts/calc_constraints.py pyproject.toml > lowerbounds_constraints.txt 68 | fi 69 | 70 | if [ -f $POST_BEFORE_INSTALL ]; then 71 | source $POST_BEFORE_INSTALL 72 | fi 73 | -------------------------------------------------------------------------------- /.github/stale.yml: -------------------------------------------------------------------------------- 1 | # WARNING: DO NOT EDIT! 2 | # 3 | # This file was generated by plugin_template, and is managed by it. Please use 4 | # './plugin-template --github pulp_rpm' to update this file. 5 | # 6 | # For more info visit https://github.com/pulp/plugin_template 7 | # Configuration for probot-stale - https://github.com/probot/stale 8 | 9 | # Number of days of inactivity before an Issue or Pull Request becomes stale 10 | daysUntilStale: 90 11 | 12 | # Number of days of inactivity before an Issue or Pull Request with the stale label is closed. 13 | # Set to false to disable. If disabled, issues still need to be closed manually, but will remain marked as stale. 14 | daysUntilClose: 30 15 | 16 | # Only issues or pull requests with all of these labels are check if stale. Defaults to `[]` (disabled) 17 | onlyLabels: [] 18 | 19 | # Issues or Pull Requests with these labels will never be considered stale. Set to `[]` to disable 20 | exemptLabels: 21 | - security 22 | - planned 23 | 24 | # Set to true to ignore issues in a project (defaults to false) 25 | exemptProjects: false 26 | 27 | # Set to true to ignore issues in a milestone (defaults to false) 28 | exemptMilestones: false 29 | 30 | # Set to true to ignore issues with an assignee (defaults to false) 31 | exemptAssignees: false 32 | 33 | # Label to use when marking as stale 34 | staleLabel: stale 35 | 36 | # Limit the number of actions per hour, from 1-30. Default is 30 37 | limitPerRun: 30 38 | # Limit to only `issues` or `pulls` 39 | only: pulls 40 | 41 | pulls: 42 | markComment: |- 43 | This pull request has been marked 'stale' due to lack of recent activity. If there is no further activity, the PR will be closed in another 30 days. Thank you for your contribution! 44 | 45 | unmarkComment: >- 46 | This pull request is no longer marked for closure. 47 | 48 | closeComment: >- 49 | This pull request has been closed due to inactivity. If you feel this is in error, please reopen the pull request or file a new PR with the relevant details. 50 | 51 | issues: 52 | markComment: |- 53 | This issue has been marked 'stale' due to lack of recent activity. If there is no further activity, the issue will be closed in another 30 days. Thank you for your contribution! 54 | 55 | unmarkComment: >- 56 | This issue is no longer marked for closure. 57 | 58 | closeComment: >- 59 | This issue has been closed due to inactivity. If you feel this is in error, please reopen the issue or file a new issue with the relevant details. 60 | -------------------------------------------------------------------------------- /pulp_rpm/app/viewsets/advisory.py: -------------------------------------------------------------------------------- 1 | from pulpcore.plugin.viewsets import ( 2 | ContentFilter, 3 | NoArtifactContentUploadViewSet, 4 | ) 5 | 6 | from pulp_rpm.app.models import ( 7 | UpdateRecord, 8 | ) 9 | from pulp_rpm.app.serializers import ( 10 | MinimalUpdateRecordSerializer, 11 | UpdateRecordSerializer, 12 | ) 13 | 14 | 15 | class UpdateRecordFilter(ContentFilter): 16 | """ 17 | FilterSet for UpdateRecord. 18 | """ 19 | 20 | class Meta: 21 | model = UpdateRecord 22 | fields = { 23 | "id": ["exact", "in"], 24 | "status": ["exact", "in", "ne"], 25 | "severity": ["exact", "in", "ne"], 26 | "type": ["exact", "in", "ne"], 27 | } 28 | 29 | 30 | class UpdateRecordViewSet(NoArtifactContentUploadViewSet): 31 | """ 32 | A ViewSet for UpdateRecord. 33 | 34 | Define endpoint name which will appear in the API endpoint for this content type. 35 | For example:: 36 | http://pulp.example.com/pulp/api/v3/content/rpm/advisories/ 37 | 38 | Also specify queryset and serializer for UpdateRecord. 39 | """ 40 | 41 | endpoint_name = "advisories" 42 | queryset = UpdateRecord.objects.all() 43 | serializer_class = UpdateRecordSerializer 44 | minimal_serializer_class = MinimalUpdateRecordSerializer 45 | filterset_class = UpdateRecordFilter 46 | 47 | # TODO: adjust this policy after upload access policy design done and in place 48 | DEFAULT_ACCESS_POLICY = { 49 | "statements": [ 50 | { 51 | "action": ["list", "retrieve"], 52 | "principal": "authenticated", 53 | "effect": "allow", 54 | }, 55 | { 56 | "action": ["create"], 57 | "principal": "authenticated", 58 | "effect": "allow", 59 | "condition": [ 60 | "has_required_repo_perms_on_upload:rpm.modify_content_rpmrepository", 61 | "has_required_repo_perms_on_upload:rpm.view_rpmrepository", 62 | ], 63 | }, 64 | { 65 | "action": ["set_label", "unset_label"], 66 | "principal": "authenticated", 67 | "effect": "allow", 68 | "condition": [ 69 | "has_model_or_domain_perms:core.manage_content_labels", 70 | ], 71 | }, 72 | ], 73 | "queryset_scoping": {"function": "scope_queryset"}, 74 | } 75 | -------------------------------------------------------------------------------- /.github/workflows/pr_checks.yml: -------------------------------------------------------------------------------- 1 | # WARNING: DO NOT EDIT! 2 | # 3 | # This file was generated by plugin_template, and is managed by it. Please use 4 | # './plugin-template --github pulp_rpm' to update this file. 5 | # 6 | # For more info visit https://github.com/pulp/plugin_template 7 | 8 | --- 9 | name: "Rpm PR static checks" 10 | on: 11 | pull_request_target: 12 | types: 13 | - "opened" 14 | - "synchronize" 15 | - "reopened" 16 | branches: 17 | - "main" 18 | - "[0-9]+.[0-9]+" 19 | 20 | # This workflow runs with elevated permissions. 21 | # Do not even think about running a single bit of code from the PR. 22 | # Static analysis should be fine however. 23 | 24 | concurrency: 25 | group: "${{ github.event.pull_request.number }}-${{ github.workflow }}" 26 | cancel-in-progress: true 27 | 28 | jobs: 29 | apply_labels: 30 | runs-on: "ubuntu-latest" 31 | name: "Label PR" 32 | permissions: 33 | pull-requests: "write" 34 | steps: 35 | - uses: "actions/checkout@v4" 36 | with: 37 | fetch-depth: 0 38 | - uses: "actions/setup-python@v5" 39 | with: 40 | python-version: "3.11" 41 | - name: "Determine PR labels" 42 | run: | 43 | pip install GitPython==3.1.42 44 | git fetch origin ${{ github.event.pull_request.head.sha }} 45 | python .ci/scripts/pr_labels.py "origin/${{ github.base_ref }}" "${{ github.event.pull_request.head.sha }}" >> "$GITHUB_ENV" 46 | - uses: "actions/github-script@v7" 47 | name: "Apply PR Labels" 48 | with: 49 | script: | 50 | const { ADD_LABELS, REMOVE_LABELS } = process.env; 51 | 52 | if (REMOVE_LABELS.length) { 53 | for await (const labelName of REMOVE_LABELS.split(",")) { 54 | try { 55 | await github.rest.issues.removeLabel({ 56 | issue_number: context.issue.number, 57 | owner: context.repo.owner, 58 | repo: context.repo.repo, 59 | name: labelName, 60 | }); 61 | } catch(err) { 62 | } 63 | } 64 | } 65 | if (ADD_LABELS.length) { 66 | await github.rest.issues.addLabels({ 67 | issue_number: context.issue.number, 68 | owner: context.repo.owner, 69 | repo: context.repo.repo, 70 | labels: ADD_LABELS.split(","), 71 | }); 72 | } 73 | ... 74 | -------------------------------------------------------------------------------- /pulp_rpm/tests/functional/api/test_acs.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from pulpcore.tests.functional.utils import PulpTaskError 4 | 5 | from pulp_rpm.tests.functional.constants import ( 6 | PULP_FIXTURES_BASE_URL, 7 | RPM_FIXTURE_SUMMARY, 8 | RPM_KICKSTART_FIXTURE_SUMMARY, 9 | RPM_KICKSTART_ONLY_META_FIXTURE_URL, 10 | RPM_ONLY_METADATA_REPO_URL, 11 | ) 12 | 13 | from pulpcore.client.pulp_rpm import ( 14 | RpmRepositorySyncURL, 15 | ) 16 | 17 | 18 | @pytest.mark.parametrize( 19 | "paths,remote_url,content_summary", 20 | [ 21 | (["rpm-unsigned/"], RPM_ONLY_METADATA_REPO_URL, RPM_FIXTURE_SUMMARY), 22 | ( 23 | ["rpm-distribution-tree/"], 24 | RPM_KICKSTART_ONLY_META_FIXTURE_URL, 25 | RPM_KICKSTART_FIXTURE_SUMMARY, 26 | ), 27 | ], 28 | ) 29 | def test_acs_simple( 30 | paths, 31 | remote_url, 32 | content_summary, 33 | rpm_repository_version_api, 34 | rpm_repository_api, 35 | rpm_acs_api, 36 | rpm_repository_factory, 37 | rpm_rpmremote_factory, 38 | get_content_summary, 39 | monitor_task, 40 | monitor_task_group, 41 | gen_object_with_cleanup, 42 | delete_orphans_pre, 43 | ): 44 | """Test to sync repo with use of ACS.""" 45 | # ACS is rpm-unsigned repository which has all packages needed 46 | acs_remote = rpm_rpmremote_factory(url=PULP_FIXTURES_BASE_URL, policy="on_demand") 47 | 48 | acs_data = { 49 | "name": "alternatecontentsource", 50 | "remote": acs_remote.pulp_href, 51 | "paths": paths, 52 | } 53 | acs = gen_object_with_cleanup(rpm_acs_api, acs_data) 54 | 55 | repo = rpm_repository_factory() 56 | remote = rpm_rpmremote_factory(url=remote_url) 57 | 58 | # Sync repo with metadata only, before ACS refresh it should fail 59 | repository_sync_data = RpmRepositorySyncURL(remote=remote.pulp_href) 60 | 61 | with pytest.raises(PulpTaskError) as ctx: 62 | sync_response = rpm_repository_api.sync(repo.pulp_href, repository_sync_data) 63 | monitor_task(sync_response.task) 64 | 65 | assert "404, message='Not Found'" in ctx.value.task.error["description"] 66 | 67 | # ACS refresh 68 | acs_refresh = rpm_acs_api.refresh(acs.pulp_href) 69 | monitor_task_group(acs_refresh.task_group) 70 | 71 | # Sync repository with metadata only 72 | sync_response = rpm_repository_api.sync(repo.pulp_href, repository_sync_data) 73 | monitor_task(sync_response.task) 74 | 75 | repo = rpm_repository_api.read(repo.pulp_href) 76 | present_summary = get_content_summary(repo)["present"] 77 | assert present_summary == content_summary 78 | -------------------------------------------------------------------------------- /docs/admin/guides/add-signing-services.md: -------------------------------------------------------------------------------- 1 | # Register Signing Services 2 | 3 | Create a `SigningService` for signing RPM metadata (`repomd.xml`) or RPM Packages. 4 | 5 | ## Metadata Signing 6 | 7 | RPM metadata signing uses detached signature, which is already provided by pulpcore. 8 | To register such a service, follow the general instructions [in pulpcore](site:pulpcore/docs/admin/guides/sign-metadata/). 9 | 10 | ## Package Signing 11 | 12 | !!! tip "New in 3.26.0 (Tech Preview)" 13 | 14 | Package signing is not detached as metadata signing, so it uses a different type of `SigningService`. 15 | Nevertheless, the process of registering is very similar. 16 | 17 | ### Pre-Requisites 18 | 19 | - Get familiar with the general SigningService registration [here](site:pulpcore/docs/admin/guides/sign-metadata/). 20 | 21 | ### Instructions 22 | 23 | 1. Create a signing script capable of signing an RPM Package. 24 | - The script receives a file path as its first argument. 25 | - The script should return a json-formatted output. No signature is required, since its embedded. 26 | ```json 27 | {"file": "filename"} 28 | ``` 29 | 1. Register it with `pulpcore-manager add-signing-service`. 30 | - The `--class` should be `rpm:RpmPackageSigningService`. 31 | - The key provided here serves only for validating the script. 32 | The signing fingerprint is provided dynamically, as [on upload signing](site:pulp_rpm/docs/user/guides/sign-packages/#on-upload). 33 | 1. Retrieve the signing service for usage. 34 | 35 | ### Example 36 | 37 | Write a signing script. 38 | The following example is roughly what we use for testing. 39 | 40 | ```bash title="package-signing-script.sh" 41 | #!/usr/bin/env bash 42 | 43 | # Input provided to the script 44 | FILE_PATH=$1 45 | FINGERPRINT="${PULP_SIGNING_KEY_FINGERPRINT}" 46 | 47 | # Specific signing logic 48 | GPG_HOME=${HOME}/.gnupg 49 | GPG_BIN=/usr/bin/gpg 50 | rpm \ 51 | --define "_signature gpg" \ 52 | --define "_gpg_path ${GPG_HOME}" \ 53 | --define "_gpg_name ${FINGERPRINT}" \ 54 | --define "_gpgbin ${GPG_BIN}" \ 55 | --addsign "${FILE_PATH}" 1> /dev/null 56 | 57 | # Output 58 | STATUS=$? 59 | if [[ ${STATUS} -eq 0 ]]; then 60 | echo {\"rpm_package\": \"${FILE_PATH}\"} 61 | else 62 | exit ${STATUS} 63 | fi 64 | ``` 65 | 66 | Register the signing service and retrieve information about it. 67 | 68 | ```bash 69 | pulpcore-manager add-signing-service \ 70 | "SimpleRpmSigningService" \ 71 | ${SCRIPT_ABS_FILENAME} \ 72 | ${KEYID} \ 73 | --class "rpm:RpmPackageSigningService" 74 | 75 | pulp signing-service show --name "SimpleRpmSigningService" 76 | ``` 77 | 78 | 79 | 80 | -------------------------------------------------------------------------------- /pulp_rpm/app/fields.py: -------------------------------------------------------------------------------- 1 | from rest_framework import serializers 2 | from pulp_rpm.app.constants import ADVISORY_SUM_TYPE_TO_NAME 3 | from pulp_rpm.app.models import UpdateReference 4 | 5 | 6 | class UpdateCollectionPackagesField(serializers.ListField): 7 | """ 8 | A serializer field for the 'UpdateCollectionPackage' model. 9 | """ 10 | 11 | child = serializers.DictField() 12 | 13 | def to_representation(self, obj): 14 | """ 15 | Get list of packages from UpdateCollections for UpdateRecord if any. 16 | 17 | Args: 18 | value ('pk' of `UpdateRecord` instance): UUID of UpdateRecord instance 19 | 20 | Returns: 21 | A list of dictionaries representing packages inside the collections of UpdateRecord 22 | 23 | """ 24 | ret = [] 25 | for pkg in obj.packages.values(): 26 | ret.append( 27 | { 28 | "arch": pkg["arch"], 29 | "epoch": pkg["epoch"], 30 | "filename": pkg["filename"], 31 | "name": pkg["name"], 32 | "reboot_suggested": pkg["reboot_suggested"], 33 | "relogin_suggested": pkg["relogin_suggested"], 34 | "restart_suggested": pkg["restart_suggested"], 35 | "release": pkg["release"], 36 | "src": pkg["src"], 37 | "sum": pkg["sum"], 38 | "sum_type": ADVISORY_SUM_TYPE_TO_NAME.get(pkg["sum_type"], ""), 39 | "version": pkg["version"], 40 | } 41 | ) 42 | 43 | return ret 44 | 45 | 46 | class UpdateReferenceField(serializers.ListField): 47 | """ 48 | A serializer field for the 'UpdateReference' model. 49 | """ 50 | 51 | child = serializers.DictField() 52 | 53 | def to_representation(self, value): 54 | """ 55 | Get list of references from UpdateReferences for UpdateRecord if any. 56 | 57 | Args: 58 | value ('pk' of `UpdateRecord` instance): UUID of UpdateRecord instance 59 | 60 | Returns: 61 | A list of dictionaries representing references inside the collections of UpdateRecord 62 | 63 | """ 64 | ret = [] 65 | references = UpdateReference.objects.filter(update_record=value) 66 | for reference in references: 67 | ret.append( 68 | { 69 | "href": reference.href, 70 | "id": reference.ref_id, 71 | "title": reference.title, 72 | "type": reference.ref_type, 73 | } 74 | ) 75 | return ret 76 | -------------------------------------------------------------------------------- /pulp_rpm/app/serializers/prune.py: -------------------------------------------------------------------------------- 1 | from gettext import gettext as _ 2 | 3 | from rest_framework import fields, serializers 4 | 5 | from pulp_rpm.app.models import RpmRepository 6 | 7 | from pulpcore.plugin.serializers import ValidateFieldsMixin 8 | from pulpcore.plugin.util import get_domain 9 | 10 | 11 | class PrunePackagesSerializer(serializers.Serializer, ValidateFieldsMixin): 12 | """ 13 | Serializer for prune-old-Packages operation. 14 | """ 15 | 16 | repo_hrefs = fields.ListField( 17 | required=True, 18 | help_text=_( 19 | "Will prune old packages from the specified list of repos. " 20 | "Use ['*'] to specify all repos. " 21 | "Will prune based on the specified repositories' latest_versions." 22 | ), 23 | child=serializers.CharField(), 24 | ) 25 | 26 | keep_days = serializers.IntegerField( 27 | help_text=_( 28 | "Prune packages introduced *prior-to* this many days ago. " 29 | "Default is 14. A value of 0 implies 'keep latest package only.'" 30 | ), 31 | required=False, 32 | min_value=0, 33 | default=14, 34 | ) 35 | 36 | dry_run = serializers.BooleanField( 37 | help_text=_( 38 | "Determine what would-be-pruned and log the list of packages. " 39 | "Intended as a debugging aid." 40 | ), 41 | default=False, 42 | required=False, 43 | ) 44 | 45 | def validate_repo_hrefs(self, value): 46 | """ 47 | Insure repo_hrefs is not empty and contains either valid RPM Repository hrefs or "*". 48 | Args: 49 | value (list): The list supplied by the user 50 | Returns: 51 | The list of RpmRepositories after validation 52 | Raises: 53 | ValidationError: If the list is empty or contains invalid hrefs. 54 | """ 55 | if len(value) == 0: 56 | raise serializers.ValidationError("Must not be [].") 57 | 58 | # prune-all-repos is "*" - find all RPM repos in this domain 59 | if "*" in value: 60 | if len(value) != 1: 61 | raise serializers.ValidationError("Can't specify specific HREFs when using '*'") 62 | return RpmRepository.objects.filter(pulp_domain=get_domain()) 63 | 64 | from pulpcore.plugin.viewsets import NamedModelViewSet 65 | 66 | # We're pruning a specific list of RPM repositories. 67 | # Validate that they are for RpmRepositories. 68 | hrefs_to_return = [] 69 | for href in value: 70 | hrefs_to_return.append(NamedModelViewSet.get_resource(href, RpmRepository)) 71 | 72 | return hrefs_to_return 73 | -------------------------------------------------------------------------------- /pulp_rpm/tests/performance/test_pulp_to_pulp.py: -------------------------------------------------------------------------------- 1 | """Tests that verify download of content served by Pulp.""" 2 | 3 | import pytest 4 | 5 | from pulp_rpm.tests.functional.constants import ( 6 | CENTOS8_STREAM_BASEOS_URL, 7 | CENTOS8_STREAM_APPSTREAM_URL, 8 | ) 9 | 10 | 11 | @pytest.mark.parallel 12 | @pytest.mark.parametrize("url", [CENTOS8_STREAM_BASEOS_URL, CENTOS8_STREAM_APPSTREAM_URL]) 13 | def test_pulp_to_pulp( 14 | url, 15 | init_and_sync, 16 | rpm_publication_factory, 17 | rpm_distribution_factory, 18 | rpm_repository_version_api, 19 | distribution_base_url, 20 | ): 21 | """Verify whether content served by pulp can be synced. 22 | 23 | Do the following: 24 | 25 | 1. Create, populate, publish, and distribute a repository. 26 | 2. Sync other repository using as remote url, 27 | the distribution base_url from the previous repository. 28 | 29 | """ 30 | repo, remote, task = init_and_sync(url=url, policy="on_demand", return_task=True) 31 | task_duration = task.finished_at - task.started_at 32 | waiting_time = task.started_at - task.pulp_created 33 | print( 34 | "\n-> Sync => Waiting time (s): {wait} | Service time (s): {service}".format( 35 | wait=waiting_time.total_seconds(), service=task_duration.total_seconds() 36 | ) 37 | ) 38 | 39 | # Create a publication & distribution 40 | publication = rpm_publication_factory(repository=repo.pulp_href) 41 | distribution = rpm_distribution_factory(publication=publication.pulp_href) 42 | 43 | # Create another repo pointing to distribution base_url 44 | repo2, remote2, task = init_and_sync( 45 | url=distribution_base_url(distribution.base_url), policy="on_demand", return_task=True 46 | ) 47 | task_duration = task.finished_at - task.started_at 48 | waiting_time = task.started_at - task.pulp_created 49 | print( 50 | "\n-> Sync => Waiting time (s): {wait} | Service time (s): {service}".format( 51 | wait=waiting_time.total_seconds(), service=task_duration.total_seconds() 52 | ) 53 | ) 54 | 55 | repo_ver = rpm_repository_version_api.read(repo.latest_version_href) 56 | repo2_ver = rpm_repository_version_api.read(repo2.latest_version_href) 57 | repo_summary = {k: v["count"] for k, v in repo_ver.content_summary.present.items()} 58 | repo2_summary = {k: v["count"] for k, v in repo2_ver.content_summary.present.items()} 59 | assert repo_summary == repo2_summary 60 | 61 | repo_summary = {k: v["count"] for k, v in repo_ver.content_summary.added.items()} 62 | repo2_summary = {k: v["count"] for k, v in repo2_ver.content_summary.added.items()} 63 | assert repo_summary == repo2_summary 64 | -------------------------------------------------------------------------------- /pulp_rpm/tests/functional/api/test_download_content.py: -------------------------------------------------------------------------------- 1 | """Tests that verify download of content served by Pulp.""" 2 | 3 | import hashlib 4 | from random import choice 5 | from urllib.parse import urljoin 6 | 7 | import pytest 8 | import requests 9 | 10 | from pulp_rpm.tests.functional.constants import RPM_UNSIGNED_FIXTURE_URL 11 | from pulp_rpm.tests.functional.utils import ( 12 | get_package_repo_path, 13 | ) 14 | from pulpcore.client.pulp_rpm import RpmRpmPublication 15 | 16 | 17 | @pytest.mark.parallel 18 | def test_all( 19 | rpm_unsigned_repo_immediate, 20 | rpm_package_api, 21 | rpm_publication_api, 22 | rpm_distribution_factory, 23 | download_content_unit, 24 | gen_object_with_cleanup, 25 | ): 26 | """Verify whether content served by pulp can be downloaded. 27 | 28 | The process of publishing content is more involved in Pulp 3 than it 29 | was under Pulp 2. Given a repository, the process is as follows: 30 | 31 | 1. Create a publication from the repository. (The latest repository 32 | version is selected if no version is specified.) A publication is a 33 | repository version plus metadata. 34 | 2. Create a distribution from the publication. The distribution defines 35 | at which URLs a publication is available, e.g. 36 | ``http://example.com/content/foo/`` and 37 | ``http://example.com/content/bar/``. 38 | 39 | Do the following: 40 | 41 | 1. Create, populate, publish, and distribute a repository. 42 | 2. Select a random content unit in the distribution. Download that 43 | content unit from Pulp, and verify that the content unit has the 44 | same checksum when fetched directly from Pulp-Fixtures. 45 | """ 46 | # Sync a Repository 47 | repo = rpm_unsigned_repo_immediate 48 | 49 | # Create a publication. 50 | publish_data = RpmRpmPublication(repository=repo.pulp_href) 51 | publication = gen_object_with_cleanup(rpm_publication_api, publish_data) 52 | 53 | # Create a distribution. 54 | distribution = rpm_distribution_factory(publication=publication.pulp_href) 55 | 56 | # Pick a content unit (of each type), and download it from both Pulp Fixtures… 57 | packages = rpm_package_api.list(repository_version=repo.latest_version_href) 58 | package_paths = [p.location_href for p in packages.results] 59 | unit_path = choice(package_paths) 60 | fixture_hash = hashlib.sha256( 61 | requests.get(urljoin(RPM_UNSIGNED_FIXTURE_URL, unit_path)).content 62 | ).hexdigest() 63 | 64 | # …and Pulp. 65 | pkg_path = get_package_repo_path(unit_path) 66 | content = download_content_unit(distribution.base_path, pkg_path) 67 | pulp_hash = hashlib.sha256(content).hexdigest() 68 | 69 | assert fixture_hash == pulp_hash 70 | -------------------------------------------------------------------------------- /pulp_rpm/app/migrations/0045_modulemd_fields.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.2.15 on 2022-08-29 16:33 2 | 3 | from django.db import migrations, models 4 | 5 | import ast 6 | import logging 7 | import yaml 8 | 9 | 10 | logger = logging.getLogger(__name__) 11 | 12 | 13 | def populate_new_fields(apps, schema_editor): 14 | """Populate new fields of modulemd.""" 15 | Modulemd = apps.get_model("rpm", "Modulemd") 16 | ModulemdDefaults = apps.get_model("rpm", "ModulemdDefaults") 17 | modules_to_update = [] 18 | 19 | # Fix issue #2786 if already tried to update to 3.18.1 20 | modulemds_to_update = [] 21 | for modulemd in Modulemd.objects.filter(snippet__startswith="b\'---"): 22 | modulemd.snippet = ast.literal_eval(modulemd.snippet).decode("utf8") 23 | modulemds_to_update.append(modulemd) 24 | Modulemd.objects.bulk_update(modulemds_to_update, ["snippet"]) 25 | 26 | # Same issue (#2786) has happened to modulemd defaults 27 | modulemd_defaults_to_update = [] 28 | for default in ModulemdDefaults.objects.filter(snippet__startswith="b\'---"): 29 | default.snippet = ast.literal_eval(default.snippet).decode("utf8") 30 | modulemd_defaults_to_update.append(default) 31 | ModulemdDefaults.objects.bulk_update(modulemd_defaults_to_update, ["snippet"]) 32 | 33 | for modulemd in Modulemd.objects.filter(profiles={}): 34 | try: 35 | modulemd_dict = yaml.safe_load(modulemd.snippet) 36 | modulemd.profiles = modulemd_dict["data"].get("profiles", {}) 37 | modulemd.description = modulemd_dict["data"].get("description", "") 38 | modules_to_update.append(modulemd) 39 | except ValueError as err: 40 | # Due to issue #2735 it could happen that snippet will be empty 41 | # https://github.com/pulp/pulp_rpm/issues/2735 42 | logger.warning( 43 | "Modulemd {}-{}-{} cannot populate new fields." 44 | "So it will miss the info about profiles and its description.".format( 45 | modulemd.name, modulemd.stream, modulemd.version 46 | ) 47 | ) 48 | 49 | Modulemd.objects.bulk_update(modules_to_update, ["profiles", "description"]) 50 | 51 | 52 | class Migration(migrations.Migration): 53 | 54 | dependencies = [ 55 | ("rpm", "0044_noartifact_modules"), 56 | ] 57 | 58 | operations = [ 59 | migrations.AddField( 60 | model_name="modulemd", 61 | name="description", 62 | field=models.TextField(default="Description"), 63 | preserve_default=False, 64 | ), 65 | migrations.AddField( 66 | model_name="modulemd", 67 | name="profiles", 68 | field=models.JSONField(default=dict), 69 | ), 70 | migrations.RunPython(populate_new_fields), 71 | ] 72 | -------------------------------------------------------------------------------- /pulp_rpm/app/migrations/0019_migrate_updatecollection_data.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.14 on 2020-08-03 19:21 2 | 3 | from django.db import migrations 4 | 5 | def rename_update_collections(apps, schema): 6 | """Insure every UpdateCollection associated with an Advisory has a unique name""" 7 | UpdateCollection = apps.get_model("rpm", "UpdateCollection") 8 | UpdateRecord = apps.get_model("rpm", "UpdateRecord") 9 | modified_collections = [] 10 | for advisory in UpdateRecord.objects.all(): 11 | names = {} 12 | for collection in advisory.collections.all(): 13 | if collection.name in names.keys(): 14 | orig_name = collection.name 15 | new_name = "{}_{}".format(orig_name, names[orig_name]) 16 | collection.name = new_name 17 | names[orig_name] += 1 18 | modified_collections.append(collection) 19 | else: 20 | names[collection.name] = 0 21 | UpdateCollection.objects.bulk_update(modified_collections, ['name']) 22 | 23 | def delete_orphan_collections(apps, schema): 24 | """Delete orphaned collections""" 25 | UpdateCollection = apps.get_model("rpm", "UpdateCollection") 26 | UpdateCollection.objects.filter(update_record=None).delete() 27 | 28 | def map_update_collection_to_update_record(apps, schema): 29 | """Point every collection to its/a/single UpdateRecord""" 30 | UpdateCollection = apps.get_model("rpm", "UpdateCollection") 31 | for collection in UpdateCollection.objects.all(): 32 | # At this point we can assume There Will Be Only One 33 | ur = collection.update_record.all().first() 34 | collection._update_record = ur 35 | collection.save() 36 | 37 | class Migration(migrations.Migration): 38 | 39 | dependencies = [ 40 | ('rpm', '0018_updatecollection__update_record'), 41 | ] 42 | 43 | # There are three data-shapes that need to be handled when moving from 44 | # many-to-many, unconstrained, to 1-to-1 with constraints: 45 | # * one UpdateCollection points to multiple UpdateRecords - new 46 | # UpdateCollections must be created. Fixed previously under #7291. 47 | # * several UpdateCollections belonging to the same UpdateRecord have the 48 | # same name - names must be made unique (rename_update_collections) 49 | # * UpdateCollections may not point to ANY UpdateRecords - orphaned 50 | # UpdateCollections must be removed (delete_orphan_collections) 51 | # Only once we fix UpdateCollection data, do we point every Collection's 52 | # _update_record at its (now-single) Advisory (map_update_collection_to_update_record) 53 | operations = [ 54 | migrations.RunPython(rename_update_collections), 55 | migrations.RunPython(delete_orphan_collections), 56 | migrations.RunPython(map_update_collection_to_update_record), 57 | ] 58 | -------------------------------------------------------------------------------- /pulp_rpm/app/tasks/signing.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | from tempfile import NamedTemporaryFile 3 | 4 | from pulpcore.plugin.models import Artifact, CreatedResource, PulpTemporaryFile, Upload, UploadChunk 5 | from pulpcore.plugin.tasking import general_create 6 | from pulpcore.plugin.util import get_url 7 | 8 | from pulp_rpm.app.models.content import RpmPackageSigningService 9 | 10 | 11 | def _save_file(fileobj, final_package): 12 | with fileobj.file.open() as fd: 13 | final_package.write(fd.read()) 14 | final_package.flush() 15 | 16 | 17 | def _save_upload(uploadobj, final_package): 18 | chunks = UploadChunk.objects.filter(upload=uploadobj).order_by("offset") 19 | for chunk in chunks: 20 | final_package.write(chunk.file.read()) 21 | chunk.file.close() 22 | final_package.flush() 23 | 24 | 25 | def sign_and_create( 26 | app_label, 27 | serializer_name, 28 | signing_service_pk, 29 | signing_fingerprint, 30 | temporary_file_pk, 31 | *args, 32 | **kwargs, 33 | ): 34 | data = kwargs.pop("data", None) 35 | context = kwargs.pop("context", {}) 36 | 37 | # Get unsigned package file and sign it 38 | package_signing_service = RpmPackageSigningService.objects.get(pk=signing_service_pk) 39 | with NamedTemporaryFile(mode="wb", dir=".", delete=False) as final_package: 40 | try: 41 | uploaded_package = PulpTemporaryFile.objects.get(pk=temporary_file_pk) 42 | _save_file(uploaded_package, final_package) 43 | except PulpTemporaryFile.DoesNotExist: 44 | uploaded_package = Upload.objects.get(pk=temporary_file_pk) 45 | _save_upload(uploaded_package, final_package) 46 | 47 | result = package_signing_service.sign( 48 | final_package.name, pubkey_fingerprint=signing_fingerprint 49 | ) 50 | signed_package_path = Path(result["rpm_package"]) 51 | if not signed_package_path.exists(): 52 | raise Exception(f"Signing script did not create the signed package: {result}") 53 | artifact = Artifact.init_and_validate(str(signed_package_path)) 54 | artifact.save() 55 | resource = CreatedResource(content_object=artifact) 56 | resource.save() 57 | uploaded_package.delete() 58 | 59 | # Create Package content 60 | data["artifact"] = get_url(artifact) 61 | # The Package serializer validation method have two branches: the signing and non-signing. 62 | # Here, the package is already signed, so we need to update the context for a proper validation. 63 | context["sign_package"] = False 64 | # The request data is immutable when there's an upload, so we can't delete the upload out of the 65 | # request data like we do for a file. Instead, we'll delete it here. 66 | if "upload" in data: 67 | del data["upload"] 68 | general_create(app_label, serializer_name, data=data, context=context, *args, **kwargs) 69 | -------------------------------------------------------------------------------- /pulp_rpm/tests/functional/api/test_repo_sizes.py: -------------------------------------------------------------------------------- 1 | import json 2 | import subprocess 3 | 4 | from pulp_rpm.tests.functional.constants import ( 5 | RPM_UNSIGNED_FIXTURE_URL, 6 | RPM_UNSIGNED_FIXTURE_SIZE, 7 | RPM_KICKSTART_FIXTURE_URL, 8 | RPM_KICKSTART_FIXTURE_SIZE, 9 | ) 10 | 11 | 12 | def test_repo_size(init_and_sync, delete_orphans_pre, monitor_task, pulpcore_bindings): 13 | """Test that RPM repos correctly report their on-disk artifact sizes.""" 14 | monitor_task(pulpcore_bindings.OrphansCleanupApi.cleanup({"orphan_protection_time": 0}).task) 15 | repo, _ = init_and_sync(url=RPM_UNSIGNED_FIXTURE_URL, policy="on_demand") 16 | 17 | cmd = ( 18 | "pulpcore-manager", 19 | "repository-size", 20 | "--repositories", 21 | repo.pulp_href, 22 | "--include-on-demand", 23 | ) 24 | run = subprocess.run(cmd, capture_output=True, check=True) 25 | out = json.loads(run.stdout) 26 | 27 | # Assert basic items of report and test on-demand sizing 28 | assert len(out) == 1 29 | report = out[0] 30 | assert report["name"] == repo.name 31 | assert report["href"] == repo.pulp_href 32 | assert report["disk-size"] == 0 33 | assert report["on-demand-size"] == RPM_UNSIGNED_FIXTURE_SIZE 34 | 35 | _, _ = init_and_sync(repository=repo, url=RPM_UNSIGNED_FIXTURE_URL, policy="immediate") 36 | run = subprocess.run(cmd, capture_output=True, check=True) 37 | report = json.loads(run.stdout)[0] 38 | assert report["disk-size"] == RPM_UNSIGNED_FIXTURE_SIZE 39 | assert report["on-demand-size"] == 0 40 | 41 | 42 | def test_kickstart_repo_size(init_and_sync, delete_orphans_pre, monitor_task, pulpcore_bindings): 43 | """Test that kickstart RPM repos correctly report their on-disk artifact sizes.""" 44 | monitor_task(pulpcore_bindings.OrphansCleanupApi.cleanup({"orphan_protection_time": 0}).task) 45 | repo, _ = init_and_sync(url=RPM_KICKSTART_FIXTURE_URL, policy="on_demand") 46 | 47 | cmd = ( 48 | "pulpcore-manager", 49 | "repository-size", 50 | "--repositories", 51 | repo.pulp_href, 52 | "--include-on-demand", 53 | ) 54 | run = subprocess.run(cmd, capture_output=True, check=True) 55 | out = json.loads(run.stdout) 56 | 57 | # Assert basic items of report and test on-demand sizing 58 | assert len(out) == 1 59 | report = out[0] 60 | assert report["name"] == repo.name 61 | assert report["href"] == repo.pulp_href 62 | assert report["disk-size"] == 2275 # One file is always downloaded 63 | assert report["on-demand-size"] == 133810 # Not all remote artifacts have sizes 64 | 65 | _, _ = init_and_sync(repository=repo, url=RPM_KICKSTART_FIXTURE_URL, policy="immediate") 66 | run = subprocess.run(cmd, capture_output=True, check=True) 67 | report = json.loads(run.stdout)[0] 68 | assert report["disk-size"] == RPM_KICKSTART_FIXTURE_SIZE 69 | assert report["on-demand-size"] == 0 70 | -------------------------------------------------------------------------------- /pulp_rpm/app/viewsets/prune.py: -------------------------------------------------------------------------------- 1 | from drf_spectacular.utils import extend_schema 2 | from django.conf import settings 3 | from rest_framework.viewsets import ViewSet 4 | 5 | from pulpcore.plugin.viewsets import TaskGroupOperationResponse 6 | from pulpcore.plugin.models import TaskGroup 7 | from pulpcore.plugin.serializers import TaskGroupOperationResponseSerializer 8 | from pulp_rpm.app.serializers import PrunePackagesSerializer 9 | from pulp_rpm.app.tasks import prune_packages 10 | from pulpcore.plugin.tasking import dispatch 11 | 12 | 13 | class PrunePackagesViewSet(ViewSet): 14 | """ 15 | Viewset for prune-old-Packages endpoint. 16 | """ 17 | 18 | serializer_class = PrunePackagesSerializer 19 | 20 | DEFAULT_ACCESS_POLICY = { 21 | "statements": [ 22 | { 23 | "action": ["prune_packages"], 24 | "principal": "authenticated", 25 | "effect": "allow", 26 | "condition": [ 27 | "has_repository_model_or_domain_or_obj_perms:rpm.modify_content_rpmrepository", 28 | "has_repository_model_or_domain_or_obj_perms:rpm.view_rpmrepository", 29 | ], 30 | }, 31 | ], 32 | } 33 | 34 | @extend_schema( 35 | description="Trigger an asynchronous old-Package-prune operation.", 36 | responses={202: TaskGroupOperationResponseSerializer}, 37 | ) 38 | def prune_packages(self, request): 39 | """ 40 | Triggers an asynchronous old-Package-purge operation. 41 | 42 | This returns a task-group that contains a "master" task that dispatches one task 43 | per repo being pruned. This allows repositories to become available for other 44 | processing as soon as their task completes, rather than having to wait for *all* 45 | repositories to be pruned. 46 | """ 47 | serializer = PrunePackagesSerializer(data=request.data) 48 | serializer.is_valid(raise_exception=True) 49 | 50 | repos = serializer.validated_data.get("repo_hrefs", []) 51 | repos_to_prune_pks = [] 52 | for repo in repos: 53 | repos_to_prune_pks.append(repo.pk) 54 | 55 | uri = "/api/v3/rpm/prune/" 56 | if settings.DOMAIN_ENABLED: 57 | uri = f"/{request.pulp_domain.name}{uri}" 58 | exclusive_resources = [uri, f"pdrn:{request.pulp_domain.pulp_id}:rpm:prune"] 59 | 60 | task_group = TaskGroup.objects.create(description="Prune old Packages.") 61 | 62 | dispatch( 63 | prune_packages, 64 | exclusive_resources=exclusive_resources, 65 | task_group=task_group, 66 | kwargs={ 67 | "repo_pks": repos_to_prune_pks, 68 | "keep_days": serializer.validated_data["keep_days"], 69 | "dry_run": serializer.validated_data["dry_run"], 70 | }, 71 | ) 72 | return TaskGroupOperationResponse(task_group, request) 73 | -------------------------------------------------------------------------------- /pulp_rpm/tests/functional/utils.py: -------------------------------------------------------------------------------- 1 | """Utilities for tests for the rpm plugin.""" 2 | 3 | import gzip 4 | import os 5 | import subprocess 6 | 7 | import pyzstd 8 | import requests 9 | 10 | from pulp_rpm.tests.functional.constants import ( 11 | PRIVATE_GPG_KEY_URL, 12 | PACKAGES_DIRECTORY, 13 | ) 14 | 15 | 16 | def gen_rpm_content_attrs(artifact, rpm_name): 17 | """Generate a dict with content unit attributes. 18 | 19 | :param artifact: A dict of info about the artifact. 20 | :returns: A semi-random dict for use in creating a content unit. 21 | """ 22 | return {"artifact": artifact.pulp_href, "relative_path": rpm_name} 23 | 24 | 25 | def init_signed_repo_configuration(): 26 | """Initialize the configuration required for verifying a signed repository. 27 | 28 | This function downloads and imports a private GPG key by invoking subprocess 29 | commands. Then, it creates a new signing service on the fly. 30 | """ 31 | # download the private key 32 | priv_key = subprocess.run( 33 | ("wget", "-q", "-O", "-", PRIVATE_GPG_KEY_URL), stdout=subprocess.PIPE 34 | ).stdout 35 | # import the downloaded private key 36 | subprocess.run(("gpg", "--import"), input=priv_key) 37 | 38 | # set the imported key to the maximum trust level 39 | key_fingerprint = "0C1A894EBB86AFAE218424CADDEF3019C2D4A8CF" 40 | completed_process = subprocess.run(("echo", f"{key_fingerprint}:6:"), stdout=subprocess.PIPE) 41 | subprocess.run(("gpg", "--import-ownertrust"), input=completed_process.stdout) 42 | 43 | # create a new signing service 44 | utils_dir_path = os.path.dirname(os.path.realpath(__file__)) 45 | signing_script_path = os.path.join(utils_dir_path, "sign-metadata.sh") 46 | 47 | return subprocess.run( 48 | ( 49 | "pulpcore-manager", 50 | "add-signing-service", 51 | "sign-metadata", 52 | f"{signing_script_path}", 53 | "pulp-fixture-signing-key", 54 | ) 55 | ) 56 | 57 | 58 | def get_package_repo_path(package_filename): 59 | """Get package repo path with directory structure. 60 | 61 | Args: 62 | package_filename(str): filename of RPM package 63 | 64 | Returns: 65 | (str): full path of RPM package in published repository 66 | 67 | """ 68 | return os.path.join(PACKAGES_DIRECTORY, package_filename.lower()[0], package_filename) 69 | 70 | 71 | def download_and_decompress_file(url): 72 | # Tests work normally but fails for S3 due '.gz' 73 | # Why is it only compressed for S3? 74 | resp = requests.get(url) 75 | decompression = None 76 | if url.endswith(".gz"): 77 | decompression = gzip.decompress 78 | elif url.endswith(".zst"): 79 | decompression = pyzstd.decompress 80 | 81 | if decompression: 82 | return decompression(resp.content) 83 | else: 84 | # FIXME: fix this as in CI primary/update_info.xml has '.gz' but it is not gzipped 85 | return resp.content 86 | -------------------------------------------------------------------------------- /pulp_rpm/app/management/commands/rpm-trim-changelogs.py: -------------------------------------------------------------------------------- 1 | from gettext import gettext as _ 2 | import sys 3 | 4 | from django.conf import settings 5 | from django.core.management import BaseCommand, CommandError 6 | 7 | from pulp_rpm.app.models import Package # noqa 8 | 9 | 10 | class Command(BaseCommand): 11 | """ 12 | Django management command for trimming changelog entries on packages in the Pulp database. 13 | 14 | RPM repositories contain a list of changelog entries for each package so that commands such 15 | as "dnf changelog" or "dnf repoquery --changelogs" can display it. Since this metadata can 16 | grow to be extremely large, typically you want to keep only a few changelogs for each package 17 | (if the package is actually installed on your system, you can view all changelogs with 18 | "rpm -qa $package --changelogs"). 19 | 20 | In pulp_rpm 3.17 a new setting was introduced to enact this changelog limit, but it is not 21 | retroactively applied to packages that are already synced. This command will do so and can 22 | save a significant amount of disk space if Pulp is being used to sync RPM content from RHEL 23 | or Oracle Linux. 24 | """ 25 | 26 | help = _(__doc__) 27 | 28 | def add_arguments(self, parser): 29 | """Set up arguments.""" 30 | parser.add_argument( 31 | "--changelog-limit", 32 | default=settings.KEEP_CHANGELOG_LIMIT, 33 | type=int, 34 | required=False, 35 | help=_( 36 | "The number of changelogs you wish to keep for each package - the suggested " 37 | "value is 10. If this value is not provided the default configured as per the " 38 | "settings will be used." 39 | ), 40 | ) 41 | 42 | def handle(self, *args, **options): 43 | """Implement the command.""" 44 | changelog_limit = options["changelog_limit"] 45 | if changelog_limit <= 0: 46 | raise CommandError("--changelog-limit must be a non-zero positive integer") 47 | trimmed_packages = 0 48 | batch = [] 49 | 50 | def update_total(total): 51 | sys.stdout.write("\rTrimmed changelogs for {} packages".format(total)) 52 | sys.stdout.flush() 53 | 54 | for package in Package.objects.all().only("changelogs").iterator(): 55 | # make sure the changelogs are ascending sorted by date 56 | package.changelogs.sort(key=lambda t: t[1]) 57 | # take the last N changelogs 58 | package.changelogs = package.changelogs[-changelog_limit:] 59 | batch.append(package) 60 | if len(batch) > 500: 61 | Package.objects.bulk_update(batch, fields=["changelogs"]) 62 | trimmed_packages += len(batch) 63 | batch.clear() 64 | update_total(trimmed_packages) 65 | 66 | Package.objects.bulk_update(batch, fields=["changelogs"]) 67 | trimmed_packages += len(batch) 68 | batch.clear() 69 | update_total(trimmed_packages) 70 | print() 71 | -------------------------------------------------------------------------------- /pulp_rpm/app/models/content.py: -------------------------------------------------------------------------------- 1 | import tempfile 2 | from pathlib import Path 3 | 4 | from django.conf import settings 5 | from pulpcore.plugin.exceptions import PulpException 6 | from pulpcore.plugin.models import SigningService 7 | from typing import Optional 8 | 9 | from pulp_rpm.app.shared_utils import RpmTool 10 | 11 | 12 | class RpmPackageSigningService(SigningService): 13 | """ 14 | A model used for signing RPM packages. 15 | 16 | The pubkey_fingerprint should be passed explicitly in the sign method. 17 | """ 18 | 19 | def _env_variables(self, env_vars=None): 20 | # Prevent the signing service pubkey to be used for signing a package. 21 | # The pubkey should be provided explicitly. 22 | _env_vars = {"PULP_SIGNING_KEY_FINGERPRINT": None} 23 | if env_vars: 24 | _env_vars.update(env_vars) 25 | return super()._env_variables(_env_vars) 26 | 27 | def sign( 28 | self, 29 | filename: str, 30 | env_vars: Optional[dict] = None, 31 | pubkey_fingerprint: Optional[str] = None, 32 | ): 33 | """ 34 | Sign a package @filename using @pubkey_figerprint. 35 | 36 | Args: 37 | filename: The absolute path to the package to be signed. 38 | env_vars: (optional) Dict of env_vars to be passed to the signing script. 39 | pubkey_fingerprint: The V4 fingerprint that correlates with the private key to use. 40 | """ 41 | if not pubkey_fingerprint: 42 | raise ValueError("A pubkey_fingerprint must be provided.") 43 | _env_vars = env_vars or {} 44 | _env_vars["PULP_SIGNING_KEY_FINGERPRINT"] = pubkey_fingerprint 45 | return super().sign(filename, _env_vars) 46 | 47 | def validate(self): 48 | """ 49 | Validate a signing service for a Rpm Package signature. 50 | 51 | Specifically, it validates that self.signing_script can sign an rpm package with 52 | the sample key self.pubkey and that the self.sign() method returns: 53 | 54 | ```json 55 | {"rpm_package": ""} 56 | ``` 57 | 58 | See [RpmTool.verify_signature][] for the signature verificaton method used. 59 | """ 60 | with tempfile.TemporaryDirectory(dir=settings.WORKING_DIRECTORY) as temp_directory_name: 61 | # get and sign sample rpm 62 | temp_file = RpmTool.get_empty_rpm(temp_directory_name) 63 | return_value = self.sign(temp_file, pubkey_fingerprint=self.pubkey_fingerprint) 64 | try: 65 | result = Path(return_value["rpm_package"]) 66 | except KeyError: 67 | raise PulpException(f"Malformed output from signing script: {return_value}") 68 | 69 | if not result.exists(): 70 | raise PulpException(f"Signed package not found: {result}") 71 | 72 | # verify with rpm tool 73 | rpm_tool = RpmTool(root=Path(temp_directory_name)) 74 | rpm_tool.import_pubkey_string(self.public_key) 75 | rpm_tool.verify_signature(result) 76 | -------------------------------------------------------------------------------- /pulp_rpm/tests/functional/api/test_auto_publish.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | """Tests that sync rpm plugin repositories.""" 3 | import pytest 4 | 5 | 6 | from pulpcore.client.pulp_rpm import ( 7 | RpmRepositorySyncURL, 8 | ) 9 | 10 | 11 | @pytest.fixture 12 | def setup_autopublish(rpm_repository_factory, rpm_rpmremote_factory, rpm_distribution_factory): 13 | """Create remote, repo, publish settings, and distribution.""" 14 | remote = rpm_rpmremote_factory() 15 | repo = rpm_repository_factory(autopublish=True, checksum_type="sha512") 16 | distribution = rpm_distribution_factory(repository=repo.pulp_href) 17 | 18 | return repo, remote, distribution 19 | 20 | 21 | @pytest.mark.parallel 22 | def test_01_sync(setup_autopublish, rpm_repository_api, rpm_publication_api, monitor_task): 23 | """Assert that syncing the repository triggers auto-publish and auto-distribution.""" 24 | repo, remote, distribution = setup_autopublish 25 | assert rpm_publication_api.list(repository=repo.pulp_href).count == 0 26 | assert distribution.publication is None 27 | 28 | # Sync the repository. 29 | repository_sync_data = RpmRepositorySyncURL(remote=remote.pulp_href) 30 | sync_response = rpm_repository_api.sync(repo.pulp_href, repository_sync_data) 31 | task = monitor_task(sync_response.task) 32 | 33 | # Check that all the appropriate resources were created 34 | assert len(task.created_resources) > 1 35 | publications = rpm_publication_api.list(repository=repo.pulp_href) 36 | assert publications.count == 1 37 | 38 | # Check that the publish settings were used 39 | publication = publications.results[0] 40 | assert publication.checksum_type == "sha512" 41 | 42 | # Sync the repository again. Since there should be no new repository version, there 43 | # should be no new publications or distributions either. 44 | sync_response = rpm_repository_api.sync(repo.pulp_href, repository_sync_data) 45 | task = monitor_task(sync_response.task) 46 | 47 | assert len(task.created_resources) == 0 48 | assert rpm_publication_api.list(repository=repo.pulp_href).count == 1 49 | 50 | 51 | @pytest.mark.parallel 52 | def test_02_modify( 53 | setup_autopublish, rpm_repository_api, rpm_package_api, rpm_publication_api, monitor_task 54 | ): 55 | """Assert that modifying the repository triggers auto-publish and auto-distribution.""" 56 | repo, remote, distribution = setup_autopublish 57 | assert rpm_publication_api.list(repository=repo.pulp_href).count == 0 58 | assert distribution.publication is None 59 | 60 | # Modify the repository by adding a content unit 61 | content = rpm_package_api.list().results[0].pulp_href 62 | 63 | modify_response = rpm_repository_api.modify(repo.pulp_href, {"add_content_units": [content]}) 64 | task = monitor_task(modify_response.task) 65 | 66 | # Check that all the appropriate resources were created 67 | assert len(task.created_resources) > 1 68 | publications = rpm_publication_api.list(repository=repo.pulp_href) 69 | assert publications.count == 1 70 | 71 | # Check that the publish settings were used 72 | publication = publications.results[0] 73 | assert publication.checksum_type == "sha512" 74 | -------------------------------------------------------------------------------- /docs/admin/reference/settings.md: -------------------------------------------------------------------------------- 1 | # Settings 2 | 3 | pulp_rpm adds configuration options to the those offered by pulpcore. 4 | 5 | ## KEEP_CHANGELOG_LIMIT 6 | 7 | This setting controls how many changelog entries (from the most recent ones) should 8 | be kept for each RPM package synced or uploaded into Pulp. The limit is enacted to 9 | avoid metadata bloat, as it can \_significantly\_ reduce the amount of space needed 10 | to store metadata, the amount of bandwidth needed to download it, and the amount of 11 | time needed to create it. 12 | 13 | The changelog metadata is used for the `dnf changelog` command, which can display the 14 | changelogs of a package even if it is not installed on the system. This setting 15 | therefore controls the maximum number of changelogs that can be viewed on clients 16 | using Pulp-hosted repositories using this command. Note, however, that for installed 17 | packages the `rpm -qa --changelog` command can show all available changelogs for that 18 | package without limitation. 19 | 20 | 10 was selected as the default because it is a good compromise between utility and 21 | efficiency - and because it is the value used by Fedora, CentOS, OpenSUSE, and others. 22 | 23 | ## ALLOW_AUTOMATIC_UNSAFE_ADVISORY_CONFLICT_RESOLUTION 24 | 25 | This setting controls whether or not pulp_rpm will block advisory sync or 26 | upload if it appears an 'incoming' advisory is incompatible with an existing 27 | advisory with the same name. 28 | 29 | This defaults to False, and advisory-merge will raise an `AdvisoryConflict` 30 | exception in two scenarios: 31 | 32 | - Situation 1 33 | 34 | Updated date and version are the same but pkglists differ (and one is not a subset 35 | or superset of the other). E.g. It's likely a mistake in one of the pkglists. 36 | 37 | - Situation 2 38 | 39 | Updated dates are different but pkglists have no intersection at all. E.g. It's 40 | either an attempt to combine content from incompatible repos (RHEL6-main and RHEL7 41 | debuginfo), or someone created a completely different advisory with already used id. 42 | 43 | If this setting is True, Pulp will merge the advisories in Situation 1, and simply 44 | accept the new advisory in Situation 2. 45 | 46 | If the setting is False, then the merge is rejected until the user has examined the 47 | conflicting advisories and addressed the problem. 48 | 49 | Addressing the problem manually could take a number of forms. Examples include 50 | (but are not limited to): 51 | 52 | - remove the existing advisory from the destination repository 53 | - choose not to sync from the offending remote 54 | - evaluate the command and choose not to combine conflicting repositories (e.g. RHEL6-main and RHEL7-debuginfo) 55 | 56 | !!! note 57 | This approach to conflict-resolution is done **AT YOUR OWN RISK**. 58 | Pulp cannot guarantee the usability/usefulness of the resulting advisory. 59 | 60 | 61 | ## RPM_METADATA_USE_REPO_PACKAGE_TIME 62 | 63 | When publishing RPM metadata, if this is true, Pulp will use the timestamp that the package was 64 | added to the repo rather than the timestamp that the package first appeared in Pulp. This timestamp 65 | appears in the "file" field of the time element for each package in primary.xml. Defaults to 66 | `False`. 67 | --------------------------------------------------------------------------------