├── .github └── workflows │ ├── new-issues-to-fedora-copr-board.yml │ ├── python-diff-lint.yml │ └── shell-diff-lint.yml ├── .gitignore ├── .packit.yaml ├── .tito ├── packages │ ├── .readme │ ├── dist-git │ └── dist-git-client ├── releasers.conf └── tito.props ├── COPYING ├── README.md ├── dist-git-client ├── LICENSE ├── README.md ├── bin │ └── dist-git-client ├── dist-git-client ├── dist-git-client.spec ├── dist_git_client.py ├── etc │ └── default.ini ├── run_tests.sh ├── tags └── tests │ ├── test_dist_git_client.py │ └── test_url_parser.py └── dist-git ├── LICENSE ├── README.md ├── Vagrantfile ├── beaker-tests ├── Makefile ├── PURPOSE ├── README.md ├── data │ └── prunerepo-1.1-1.fc23.src.rpm ├── files │ └── etc │ │ ├── rpkg.conf │ │ └── rpkg │ │ ├── fedpkg.conf │ │ └── rhpkg.conf ├── pkgs-files │ ├── lookaside-upload.conf │ └── ssl.conf ├── run.sh ├── setup.sh └── tests │ ├── basic-test │ ├── dist-git.conf │ └── run.sh │ ├── direct-test │ ├── dist-git-no-namespace.conf │ ├── dist-git.conf │ └── run.sh │ ├── fedpkg-lookaside_dir-test │ ├── dist-git.conf │ └── run.sh │ ├── fedpkg-test │ ├── dist-git.conf │ └── run.sh │ ├── group-check-test │ ├── dist-git-group-check-off.conf │ ├── dist-git-group-check-on.conf │ ├── dist-git-group-check-unset.conf │ └── run.sh │ ├── rhpkg-test.disable │ ├── dist-git.conf │ └── run.sh │ └── test-template │ └── run.sh ├── configs ├── dist-git │ └── dist-git.conf ├── httpd │ ├── dist-git.conf │ └── dist-git │ │ ├── git-smart-http.conf │ │ ├── lookaside-upload.conf.example │ │ ├── lookaside.conf │ │ └── manifest.conf ├── systemd │ ├── dist-git-gc.service │ ├── dist-git-gc.timer │ ├── dist-git.socket │ └── dist-git@.service └── sysusers.d │ └── dist-git.conf ├── dist-git.spec ├── docs └── scripts │ └── httpd │ ├── dist-git-stg.conf │ └── upload-rh.cgi ├── images ├── server-communication.png ├── storage.png └── tutorial.png ├── scripts ├── dist-git │ ├── dist-git-gc │ ├── hooks │ │ ├── grok_update │ │ └── post-receive │ ├── mkbranch │ ├── mkbranch_branching │ ├── remove_unused_sources │ └── setup_git_package └── httpd │ └── upload.cgi ├── selinux ├── dist_git.fc ├── dist_git.if └── dist_git.te └── tests └── test_upload_script.py /.github/workflows/new-issues-to-fedora-copr-board.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: Move new issues to issues review 3 | 4 | on: 5 | issues: 6 | types: [opened] 7 | 8 | jobs: 9 | move-new-issue: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/add-to-project@v1.0.2 13 | with: 14 | project-url: https://github.com/orgs/fedora-copr/projects/1 15 | github-token: ${{ secrets.ADD_TO_COPR_KANBAN_TOKEN }} 16 | -------------------------------------------------------------------------------- /.github/workflows/python-diff-lint.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: Lint Python issues 3 | 4 | on: 5 | pull_request: 6 | branches: [main] 7 | 8 | jobs: 9 | python-lint-job: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - name: Repository checkout 13 | uses: actions/checkout@v3 14 | 15 | - name: VCS Diff Lint 16 | uses: fedora-copr/vcs-diff-lint-action@v1 17 | -------------------------------------------------------------------------------- /.github/workflows/shell-diff-lint.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: Lint Shell issues 3 | 4 | on: 5 | pull_request: 6 | branches: [main] 7 | 8 | permissions: 9 | contents: read 10 | 11 | jobs: 12 | shell-lint-job: 13 | runs-on: ubuntu-latest 14 | 15 | steps: 16 | - name: Repository checkout 17 | uses: actions/checkout@v3 18 | with: 19 | fetch-depth: 0 20 | 21 | - name: Differential ShellCheck 22 | uses: redhat-plumbers-in-action/differential-shellcheck@v3 23 | with: 24 | token: ${{ secrets.GITHUB_TOKEN }} 25 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.swp 2 | .vagrant 3 | -------------------------------------------------------------------------------- /.packit.yaml: -------------------------------------------------------------------------------- 1 | # See the documentation for more information: 2 | # https://packit.dev/docs/configuration/ 3 | --- 4 | upstream_project_url: https://github.com/release-engineering/dist-git 5 | 6 | actions: &common_actions 7 | create-archive: 8 | - bash -c "tito build --tgz --test -o ." 9 | - bash -c "ls -1t ./*.tar.gz | head -n 1" 10 | get-current-version: 11 | - bash -c "grep -Po 'Version. +\K.*' *.spec" 12 | 13 | srpm_build_deps: 14 | - tito 15 | - git 16 | 17 | packages: 18 | dist-git: 19 | specfile_path: dist-git.spec 20 | upstream_package_name: dist-git 21 | downstream_package_name: dist-git 22 | upstream_tag_template: 'dist-git-{version}' 23 | paths: 24 | - dist-git 25 | dist-git-client: 26 | specfile_path: dist-git-client.spec 27 | upstream_package_name: dist-git-client 28 | downstream_package_name: dist-git-client 29 | upstream_tag_template: 'dist-git-client-{version}' 30 | paths: 31 | - dist-git-client 32 | jobs: 33 | - &copr 34 | job: copr_build 35 | packages: 36 | - dist-git 37 | - dist-git-client 38 | trigger: pull_request 39 | targets: 40 | - fedora-all-x86_64 41 | - epel-8-x86_64 42 | - epel-9-x86_64 43 | - epel-10-x86_64 44 | 45 | - <<: *copr 46 | trigger: commit 47 | owner: "@copr" 48 | project: "copr-dev" 49 | branch: main 50 | -------------------------------------------------------------------------------- /.tito/packages/.readme: -------------------------------------------------------------------------------- 1 | the .tito/packages directory contains metadata files 2 | named after their packages. Each file has the latest tagged 3 | version and the project's relative directory. 4 | -------------------------------------------------------------------------------- /.tito/packages/dist-git: -------------------------------------------------------------------------------- 1 | 1.18-1 dist-git/ 2 | -------------------------------------------------------------------------------- /.tito/packages/dist-git-client: -------------------------------------------------------------------------------- 1 | 1.1-1 dist-git-client/ 2 | -------------------------------------------------------------------------------- /.tito/releasers.conf: -------------------------------------------------------------------------------- 1 | [fedora] 2 | releaser = tito.release.FedoraGitReleaser 3 | branches = fedora-all epel-all 4 | -------------------------------------------------------------------------------- /.tito/tito.props: -------------------------------------------------------------------------------- 1 | [buildconfig] 2 | builder = tito.builder.Builder 3 | tagger = tito.tagger.VersionTagger 4 | changelog_do_not_remove_cherrypick = 0 5 | changelog_format = %s 6 | test_version_suffix = .post1 7 | -------------------------------------------------------------------------------- /COPYING: -------------------------------------------------------------------------------- 1 | The server part (the dist-git sub-directory) is licensed under the MIT license 2 | with the following exceptions: 3 | 4 | scripts/httpd/upload.cgi is under GPLv1 5 | 6 | The client part (the dist-git-client sub-directory) is GPLv2+. 7 | 8 | Copies of all the licenses mentioned here are distributed with the project 9 | release tarballs. 10 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | dist-git/README.md -------------------------------------------------------------------------------- /dist-git-client/LICENSE: -------------------------------------------------------------------------------- 1 | GNU GENERAL PUBLIC LICENSE 2 | Version 2, June 1991 3 | 4 | Copyright (C) 1989, 1991 Free Software Foundation, Inc., 5 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 6 | Everyone is permitted to copy and distribute verbatim copies 7 | of this license document, but changing it is not allowed. 8 | 9 | Preamble 10 | 11 | The licenses for most software are designed to take away your 12 | freedom to share and change it. By contrast, the GNU General Public 13 | License is intended to guarantee your freedom to share and change free 14 | software--to make sure the software is free for all its users. This 15 | General Public License applies to most of the Free Software 16 | Foundation's software and to any other program whose authors commit to 17 | using it. (Some other Free Software Foundation software is covered by 18 | the GNU Lesser General Public License instead.) You can apply it to 19 | your programs, too. 20 | 21 | When we speak of free software, we are referring to freedom, not 22 | price. Our General Public Licenses are designed to make sure that you 23 | have the freedom to distribute copies of free software (and charge for 24 | this service if you wish), that you receive source code or can get it 25 | if you want it, that you can change the software or use pieces of it 26 | in new free programs; and that you know you can do these things. 27 | 28 | To protect your rights, we need to make restrictions that forbid 29 | anyone to deny you these rights or to ask you to surrender the rights. 30 | These restrictions translate to certain responsibilities for you if you 31 | distribute copies of the software, or if you modify it. 32 | 33 | For example, if you distribute copies of such a program, whether 34 | gratis or for a fee, you must give the recipients all the rights that 35 | you have. You must make sure that they, too, receive or can get the 36 | source code. And you must show them these terms so they know their 37 | rights. 38 | 39 | We protect your rights with two steps: (1) copyright the software, and 40 | (2) offer you this license which gives you legal permission to copy, 41 | distribute and/or modify the software. 42 | 43 | Also, for each author's protection and ours, we want to make certain 44 | that everyone understands that there is no warranty for this free 45 | software. If the software is modified by someone else and passed on, we 46 | want its recipients to know that what they have is not the original, so 47 | that any problems introduced by others will not reflect on the original 48 | authors' reputations. 49 | 50 | Finally, any free program is threatened constantly by software 51 | patents. We wish to avoid the danger that redistributors of a free 52 | program will individually obtain patent licenses, in effect making the 53 | program proprietary. To prevent this, we have made it clear that any 54 | patent must be licensed for everyone's free use or not licensed at all. 55 | 56 | The precise terms and conditions for copying, distribution and 57 | modification follow. 58 | 59 | GNU GENERAL PUBLIC LICENSE 60 | TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 61 | 62 | 0. This License applies to any program or other work which contains 63 | a notice placed by the copyright holder saying it may be distributed 64 | under the terms of this General Public License. The "Program", below, 65 | refers to any such program or work, and a "work based on the Program" 66 | means either the Program or any derivative work under copyright law: 67 | that is to say, a work containing the Program or a portion of it, 68 | either verbatim or with modifications and/or translated into another 69 | language. (Hereinafter, translation is included without limitation in 70 | the term "modification".) Each licensee is addressed as "you". 71 | 72 | Activities other than copying, distribution and modification are not 73 | covered by this License; they are outside its scope. The act of 74 | running the Program is not restricted, and the output from the Program 75 | is covered only if its contents constitute a work based on the 76 | Program (independent of having been made by running the Program). 77 | Whether that is true depends on what the Program does. 78 | 79 | 1. You may copy and distribute verbatim copies of the Program's 80 | source code as you receive it, in any medium, provided that you 81 | conspicuously and appropriately publish on each copy an appropriate 82 | copyright notice and disclaimer of warranty; keep intact all the 83 | notices that refer to this License and to the absence of any warranty; 84 | and give any other recipients of the Program a copy of this License 85 | along with the Program. 86 | 87 | You may charge a fee for the physical act of transferring a copy, and 88 | you may at your option offer warranty protection in exchange for a fee. 89 | 90 | 2. You may modify your copy or copies of the Program or any portion 91 | of it, thus forming a work based on the Program, and copy and 92 | distribute such modifications or work under the terms of Section 1 93 | above, provided that you also meet all of these conditions: 94 | 95 | a) You must cause the modified files to carry prominent notices 96 | stating that you changed the files and the date of any change. 97 | 98 | b) You must cause any work that you distribute or publish, that in 99 | whole or in part contains or is derived from the Program or any 100 | part thereof, to be licensed as a whole at no charge to all third 101 | parties under the terms of this License. 102 | 103 | c) If the modified program normally reads commands interactively 104 | when run, you must cause it, when started running for such 105 | interactive use in the most ordinary way, to print or display an 106 | announcement including an appropriate copyright notice and a 107 | notice that there is no warranty (or else, saying that you provide 108 | a warranty) and that users may redistribute the program under 109 | these conditions, and telling the user how to view a copy of this 110 | License. (Exception: if the Program itself is interactive but 111 | does not normally print such an announcement, your work based on 112 | the Program is not required to print an announcement.) 113 | 114 | These requirements apply to the modified work as a whole. If 115 | identifiable sections of that work are not derived from the Program, 116 | and can be reasonably considered independent and separate works in 117 | themselves, then this License, and its terms, do not apply to those 118 | sections when you distribute them as separate works. But when you 119 | distribute the same sections as part of a whole which is a work based 120 | on the Program, the distribution of the whole must be on the terms of 121 | this License, whose permissions for other licensees extend to the 122 | entire whole, and thus to each and every part regardless of who wrote it. 123 | 124 | Thus, it is not the intent of this section to claim rights or contest 125 | your rights to work written entirely by you; rather, the intent is to 126 | exercise the right to control the distribution of derivative or 127 | collective works based on the Program. 128 | 129 | In addition, mere aggregation of another work not based on the Program 130 | with the Program (or with a work based on the Program) on a volume of 131 | a storage or distribution medium does not bring the other work under 132 | the scope of this License. 133 | 134 | 3. You may copy and distribute the Program (or a work based on it, 135 | under Section 2) in object code or executable form under the terms of 136 | Sections 1 and 2 above provided that you also do one of the following: 137 | 138 | a) Accompany it with the complete corresponding machine-readable 139 | source code, which must be distributed under the terms of Sections 140 | 1 and 2 above on a medium customarily used for software interchange; or, 141 | 142 | b) Accompany it with a written offer, valid for at least three 143 | years, to give any third party, for a charge no more than your 144 | cost of physically performing source distribution, a complete 145 | machine-readable copy of the corresponding source code, to be 146 | distributed under the terms of Sections 1 and 2 above on a medium 147 | customarily used for software interchange; or, 148 | 149 | c) Accompany it with the information you received as to the offer 150 | to distribute corresponding source code. (This alternative is 151 | allowed only for noncommercial distribution and only if you 152 | received the program in object code or executable form with such 153 | an offer, in accord with Subsection b above.) 154 | 155 | The source code for a work means the preferred form of the work for 156 | making modifications to it. For an executable work, complete source 157 | code means all the source code for all modules it contains, plus any 158 | associated interface definition files, plus the scripts used to 159 | control compilation and installation of the executable. However, as a 160 | special exception, the source code distributed need not include 161 | anything that is normally distributed (in either source or binary 162 | form) with the major components (compiler, kernel, and so on) of the 163 | operating system on which the executable runs, unless that component 164 | itself accompanies the executable. 165 | 166 | If distribution of executable or object code is made by offering 167 | access to copy from a designated place, then offering equivalent 168 | access to copy the source code from the same place counts as 169 | distribution of the source code, even though third parties are not 170 | compelled to copy the source along with the object code. 171 | 172 | 4. You may not copy, modify, sublicense, or distribute the Program 173 | except as expressly provided under this License. Any attempt 174 | otherwise to copy, modify, sublicense or distribute the Program is 175 | void, and will automatically terminate your rights under this License. 176 | However, parties who have received copies, or rights, from you under 177 | this License will not have their licenses terminated so long as such 178 | parties remain in full compliance. 179 | 180 | 5. You are not required to accept this License, since you have not 181 | signed it. However, nothing else grants you permission to modify or 182 | distribute the Program or its derivative works. These actions are 183 | prohibited by law if you do not accept this License. Therefore, by 184 | modifying or distributing the Program (or any work based on the 185 | Program), you indicate your acceptance of this License to do so, and 186 | all its terms and conditions for copying, distributing or modifying 187 | the Program or works based on it. 188 | 189 | 6. Each time you redistribute the Program (or any work based on the 190 | Program), the recipient automatically receives a license from the 191 | original licensor to copy, distribute or modify the Program subject to 192 | these terms and conditions. You may not impose any further 193 | restrictions on the recipients' exercise of the rights granted herein. 194 | You are not responsible for enforcing compliance by third parties to 195 | this License. 196 | 197 | 7. If, as a consequence of a court judgment or allegation of patent 198 | infringement or for any other reason (not limited to patent issues), 199 | conditions are imposed on you (whether by court order, agreement or 200 | otherwise) that contradict the conditions of this License, they do not 201 | excuse you from the conditions of this License. If you cannot 202 | distribute so as to satisfy simultaneously your obligations under this 203 | License and any other pertinent obligations, then as a consequence you 204 | may not distribute the Program at all. For example, if a patent 205 | license would not permit royalty-free redistribution of the Program by 206 | all those who receive copies directly or indirectly through you, then 207 | the only way you could satisfy both it and this License would be to 208 | refrain entirely from distribution of the Program. 209 | 210 | If any portion of this section is held invalid or unenforceable under 211 | any particular circumstance, the balance of the section is intended to 212 | apply and the section as a whole is intended to apply in other 213 | circumstances. 214 | 215 | It is not the purpose of this section to induce you to infringe any 216 | patents or other property right claims or to contest validity of any 217 | such claims; this section has the sole purpose of protecting the 218 | integrity of the free software distribution system, which is 219 | implemented by public license practices. Many people have made 220 | generous contributions to the wide range of software distributed 221 | through that system in reliance on consistent application of that 222 | system; it is up to the author/donor to decide if he or she is willing 223 | to distribute software through any other system and a licensee cannot 224 | impose that choice. 225 | 226 | This section is intended to make thoroughly clear what is believed to 227 | be a consequence of the rest of this License. 228 | 229 | 8. If the distribution and/or use of the Program is restricted in 230 | certain countries either by patents or by copyrighted interfaces, the 231 | original copyright holder who places the Program under this License 232 | may add an explicit geographical distribution limitation excluding 233 | those countries, so that distribution is permitted only in or among 234 | countries not thus excluded. In such case, this License incorporates 235 | the limitation as if written in the body of this License. 236 | 237 | 9. The Free Software Foundation may publish revised and/or new versions 238 | of the General Public License from time to time. Such new versions will 239 | be similar in spirit to the present version, but may differ in detail to 240 | address new problems or concerns. 241 | 242 | Each version is given a distinguishing version number. If the Program 243 | specifies a version number of this License which applies to it and "any 244 | later version", you have the option of following the terms and conditions 245 | either of that version or of any later version published by the Free 246 | Software Foundation. If the Program does not specify a version number of 247 | this License, you may choose any version ever published by the Free Software 248 | Foundation. 249 | 250 | 10. If you wish to incorporate parts of the Program into other free 251 | programs whose distribution conditions are different, write to the author 252 | to ask for permission. For software which is copyrighted by the Free 253 | Software Foundation, write to the Free Software Foundation; we sometimes 254 | make exceptions for this. Our decision will be guided by the two goals 255 | of preserving the free status of all derivatives of our free software and 256 | of promoting the sharing and reuse of software generally. 257 | 258 | NO WARRANTY 259 | 260 | 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY 261 | FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN 262 | OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES 263 | PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED 264 | OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 265 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS 266 | TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE 267 | PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, 268 | REPAIR OR CORRECTION. 269 | 270 | 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 271 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR 272 | REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, 273 | INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING 274 | OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED 275 | TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY 276 | YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER 277 | PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE 278 | POSSIBILITY OF SUCH DAMAGES. 279 | 280 | END OF TERMS AND CONDITIONS 281 | 282 | How to Apply These Terms to Your New Programs 283 | 284 | If you develop a new program, and you want it to be of the greatest 285 | possible use to the public, the best way to achieve this is to make it 286 | free software which everyone can redistribute and change under these terms. 287 | 288 | To do so, attach the following notices to the program. It is safest 289 | to attach them to the start of each source file to most effectively 290 | convey the exclusion of warranty; and each file should have at least 291 | the "copyright" line and a pointer to where the full notice is found. 292 | 293 | 294 | Copyright (C) 295 | 296 | This program is free software; you can redistribute it and/or modify 297 | it under the terms of the GNU General Public License as published by 298 | the Free Software Foundation; either version 2 of the License, or 299 | (at your option) any later version. 300 | 301 | This program is distributed in the hope that it will be useful, 302 | but WITHOUT ANY WARRANTY; without even the implied warranty of 303 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 304 | GNU General Public License for more details. 305 | 306 | You should have received a copy of the GNU General Public License along 307 | with this program; if not, write to the Free Software Foundation, Inc., 308 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 309 | 310 | Also add information on how to contact you by electronic and paper mail. 311 | 312 | If the program is interactive, make it output a short notice like this 313 | when it starts in an interactive mode: 314 | 315 | Gnomovision version 69, Copyright (C) year name of author 316 | Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. 317 | This is free software, and you are welcome to redistribute it 318 | under certain conditions; type `show c' for details. 319 | 320 | The hypothetical commands `show w' and `show c' should show the appropriate 321 | parts of the General Public License. Of course, the commands you use may 322 | be called something other than `show w' and `show c'; they could even be 323 | mouse-clicks or menu items--whatever suits your program. 324 | 325 | You should also get your employer (if you work as a programmer) or your 326 | school, if any, to sign a "copyright disclaimer" for the program, if 327 | necessary. Here is a sample; alter the names: 328 | 329 | Yoyodyne, Inc., hereby disclaims all copyright interest in the program 330 | `Gnomovision' (which makes passes at compilers) written by James Hacker. 331 | 332 | , 1 April 1989 333 | Ty Coon, President of Vice 334 | 335 | This General Public License does not permit incorporating your program into 336 | proprietary programs. If your program is a subroutine library, you may 337 | consider it more useful to permit linking proprietary applications with the 338 | library. If this is what you want to do, use the GNU Lesser General 339 | Public License instead of this License. 340 | -------------------------------------------------------------------------------- /dist-git-client/README.md: -------------------------------------------------------------------------------- 1 | Get sources from DistGit 2 | ======================== 3 | 4 | A simple, configurable python utility that is able to clone package sources from 5 | a DistGit repository, download sources from the corresponding lookaside cache 6 | location, and generate source RPM. 7 | 8 | The utility is able to automatically map the .git/config clone URL into the 9 | corresponding DistGit instance configuration. 10 | 11 | 12 | Usage 13 | ----- 14 | 15 | Building package from DistGit is as simple as: 16 | 17 | $ dist-git-client clone tar 18 | INFO: Checked call: git clone https://src.fedoraproject.org/rpms/tar.git 19 | Cloning into 'tar'... 20 | ... 21 | Resolving deltas: 100% (802/802), done. 22 | 23 | $ cd tar 24 | 25 | $ dist-git-client sources 26 | INFO: Reading stdout from command: git rev-parse --abbrev-ref HEAD 27 | INFO: Reading sources specification file: sources 28 | INFO: Downloading tar-1.35.tar.xz 29 | INFO: Reading stdout from command: curl --help all 30 | INFO: Calling: curl -H Pragma: -o tar-1.35.tar.xz --location --connect-timeout 60 --retry 3 --retry-delay 10 --remote-time --show-error --fail --retry-all-errors https://src.fedoraproject.org/repo/pkgs/rpms/tar/tar-1.35.tar.xz/sha512/8b84ed661e6c878fa33eb5c1808d20351e6f40551ac63f96014fb0d0b9c72d5d94d8865d39e36bcb184fd250f84778a3b271bbd8bd2ceb69eece0c3568577510/tar-1.35.tar.xz 31 | % Total % Received % Xferd Average Speed Time Time Time Current 32 | Dload Upload Total Spent Left Speed 33 | 100 2262k 100 2262k 0 0 1983k 0 0:00:01 0:00:01 --:--:-- 1984k 34 | INFO: Reading stdout from command: sha512sum tar-1.35.tar.xz 35 | INFO: Downloading tar-1.35.tar.xz.sig 36 | INFO: Calling: curl -H Pragma: -o tar-1.35.tar.xz.sig --location --connect-timeout 60 --retry 3 --retry-delay 10 --remote-time --show-error --fail --retry-all-errors https://src.fedoraproject.org/repo/pkgs/rpms/tar/tar-1.35.tar.xz.sig/sha512/00e5c95bf8015f75f59556a82ed7f50bddefe89754c7ff3c19411aee2f37626a5d65c33e18b87f7f8f96388d3f175fd095917419a3ad1c0fc9d6188088bac944/tar-1.35.tar.xz.sig 37 | % Total % Received % Xferd Average Speed Time Time Time Current 38 | Dload Upload Total Spent Left Speed 39 | 100 95 100 95 0 0 270 0 --:--:-- --:--:-- --:--:-- 270 40 | INFO: Reading stdout from command: sha512sum tar-1.35.tar.xz.sig 41 | 42 | $ dist-git-client srpm 43 | ... 44 | Wrote: /tmp/tar-1.35-3.src.rpm 45 | 46 | $ mock /tmp/tar-1.35-3.src.rpm 47 | ... 48 | Wrote: /builddir/build/RPMS/tar-debuginfo-1.35-3.fc39.x86_64.rpm 49 | Wrote: /builddir/build/RPMS/tar-1.35-3.fc39.x86_64.rpm 50 | Wrote: /builddir/build/RPMS/tar-debugsource-1.35-3.fc39.x86_64.rpm 51 | ... 52 | Finish: run 53 | 54 | 55 | Configuration 56 | ------------- 57 | 58 | The project provides also a predefined set of existing (public) DistGit 59 | instances (Fedora, CentOS, Fedora Copr). Check the configuration file 60 | `/etc/dist-git-client/default.ini` out for the configuration documentation, 61 | and feel free to drop another INI configuration file into the 62 | `/etc/dist-git-client/` directory. 63 | 64 | 65 | Install 66 | ------- 67 | 68 | From regular Fedora/EPEL repositories: 69 | 70 | $ dnf install -y dist-git-client 71 | 72 | From source code: 73 | 74 | $ tito build --rpm --install 75 | 76 | Without installation - directly from Git (development/debugging use-case, no support): 77 | 78 | $ ./dist-git-client .... 79 | -------------------------------------------------------------------------------- /dist-git-client/bin/dist-git-client: -------------------------------------------------------------------------------- 1 | #! /usr/bin/python3 2 | 3 | """ 4 | From within a git checkout, try to download files from dist-git lookaside cache. 5 | """ 6 | 7 | from dist_git_client import main 8 | 9 | if __name__ == "__main__": 10 | main() 11 | -------------------------------------------------------------------------------- /dist-git-client/dist-git-client: -------------------------------------------------------------------------------- 1 | #! /bin/sh 2 | 3 | # Run copr-disgit-client script directly from git, TESTING ONLY SCRIPT! 4 | # Copyright (C) 2020 Red Hat, Inc. 5 | # 6 | # This program is free software; you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation; either version 2 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License along 17 | # with this program; if not, write to the Free Software Foundation, Inc., 18 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 19 | 20 | absdir="$(dirname "$(readlink -f "$0")")" 21 | export PYTHONPATH="$absdir${PYTHONPATH+:$PYTHONPATH}" 22 | python3 "$absdir/bin/dist-git-client" "$@" 23 | -------------------------------------------------------------------------------- /dist-git-client/dist-git-client.spec: -------------------------------------------------------------------------------- 1 | Name: dist-git-client 2 | Version: 1.1 3 | Release: 1%{?dist} 4 | Summary: Get sources for RPM builds from DistGit repositories 5 | BuildArch: noarch 6 | 7 | License: GPL-2.0-or-later 8 | URL: https://github.com/release-engineering/dist-git.git 9 | 10 | # Source is created by 11 | # git clone https://github.com/release-engineering/dist-git.git 12 | # cd dist-git-client 13 | # tito build --tgz 14 | Source0: %{name}-%{version}.tar.gz 15 | 16 | Requires: curl 17 | Requires: /usr/bin/git 18 | 19 | BuildRequires: python3-devel 20 | BuildRequires: python3-pytest 21 | BuildRequires: python3-rpm-macros 22 | BuildRequires: /usr/bin/argparse-manpage 23 | BuildRequires: /usr/bin/git 24 | 25 | %if 0%{?fedora} || 0%{?rhel} > 9 26 | Requires: python3-rpmautospec 27 | BuildRequires: python3-rpmautospec 28 | %endif 29 | 30 | 31 | %description 32 | A simple, configurable python utility that is able to clone package sources from 33 | a DistGit repository, download sources from the corresponding lookaside cache 34 | locations, and generate source RPMs. 35 | 36 | The utility is able to automatically map the .git/config clone URL into the 37 | corresponding DistGit instance configuration. 38 | 39 | 40 | %prep 41 | %setup -q 42 | 43 | 44 | %build 45 | argparse-manpage --pyfile dist_git_client.py \ 46 | --function _get_argparser \ 47 | --author "Copr Team" \ 48 | --author-email "copr-team@redhat.com" \ 49 | --url %url --project-name Copr \ 50 | > dist-git-client.1 51 | 52 | 53 | %install 54 | install -d %{buildroot}%{_bindir} 55 | install -d %{buildroot}%{_mandir}/man1 56 | install -d %{buildroot}%{_sysconfdir}/dist-git-client 57 | install -d %{buildroot}%{python3_sitelib} 58 | install -p -m 755 bin/dist-git-client %{buildroot}%{_bindir} 59 | install -p -m 644 etc/default.ini \ 60 | %{buildroot}%{_sysconfdir}/dist-git-client 61 | install -p -m 644 dist_git_client.py %{buildroot}%{python3_sitelib} 62 | install -p -m 644 dist-git-client.1 %{buildroot}%{_mandir}/man1/ 63 | 64 | 65 | %check 66 | PYTHON=python3 ./run_tests.sh -vv --no-coverage 67 | 68 | 69 | %files 70 | %license LICENSE 71 | %doc README.md 72 | %{_bindir}/dist-git-client 73 | %{_mandir}/man1/dist-git-client.1* 74 | %dir %{_sysconfdir}/dist-git-client 75 | %config(noreplace) %{_sysconfdir}/dist-git-client/default.ini 76 | %{python3_sitelib}/dist_git_client.* 77 | %{python3_sitelib}/__pycache__/dist_git_client* 78 | 79 | 80 | %changelog 81 | * Fri Apr 04 2025 Pavel Raiskup 1.1-1 82 | - more powerful parsing for ssh clone urls 83 | - nicer error output for not-matched origin.url hostnames 84 | - apply Fedora Review fixes 85 | 86 | * Fri Jun 21 2024 Pavel Raiskup - 1.0-2 87 | - Fedora Review fixes (rhbz#2293067) 88 | 89 | * Thu Jun 06 2024 Pavel Raiskup - 1.0-1 90 | - new package built with tito 91 | -------------------------------------------------------------------------------- /dist-git-client/dist_git_client.py: -------------------------------------------------------------------------------- 1 | """ 2 | dist-git-client code, moved to a python module to simplify unit-testing 3 | """ 4 | 5 | import argparse 6 | from collections import namedtuple 7 | import configparser 8 | import errno 9 | import glob 10 | import logging 11 | import shlex 12 | import os 13 | import shutil 14 | import subprocess 15 | import sys 16 | from urllib.parse import urlparse 17 | 18 | try: 19 | from rpmautospec import ( 20 | specfile_uses_rpmautospec as rpmautospec_used, 21 | process_distgit as rpmautospec_expand, 22 | ) 23 | except ImportError: 24 | rpmautospec_used = lambda _: False 25 | 26 | 27 | def log_cmd(command, comment="Running command"): 28 | """ Dump the command to stderr so it can be c&p to shell """ 29 | command = ' '.join([shlex.quote(x) for x in command]) 30 | logging.info("%s: %s", comment, command) 31 | 32 | 33 | def check_output(cmd, comment="Reading stdout from command"): 34 | """ el6 compatible subprocess.check_output() """ 35 | log_cmd(cmd, comment) 36 | process = subprocess.Popen( 37 | cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 38 | (stdout, _) = process.communicate() 39 | if process.returncode: 40 | raise RuntimeError("Exit non-zero: {0}".format(process.returncode)) 41 | return stdout 42 | 43 | def call(cmd, comment="Calling"): 44 | """ wrap sp.call() with logging info """ 45 | log_cmd(cmd, comment) 46 | return subprocess.call(cmd) 47 | 48 | def check_call(cmd, comment="Checked call"): 49 | """ wrap sp.check_call() with logging info """ 50 | log_cmd(cmd, comment) 51 | subprocess.check_call(cmd) 52 | 53 | def _load_config(directory): 54 | config = configparser.ConfigParser() 55 | files = glob.glob(os.path.join(directory, "*.ini")) 56 | logging.debug("Files %s in config directory %s", files, directory) 57 | config.read(files) 58 | 59 | config_dict = { 60 | "instances": {}, 61 | "clone_host_map": {}, 62 | } 63 | 64 | instances = config_dict["instances"] 65 | for section_name in config.sections(): 66 | section = config[section_name] 67 | instance = instances[section_name] = {} 68 | for key in section.keys(): 69 | # array-like config options 70 | if key in ["clone_hostnames", "path_prefixes"]: 71 | hostnames = section[key].split() 72 | instance[key] = [h.strip() for h in hostnames] 73 | else: 74 | instance[key] = section[key] 75 | 76 | for key in ["sources", "specs"]: 77 | if key in instance: 78 | continue 79 | instance[key] = "." 80 | 81 | if "sources_file" not in instance: 82 | instance["sources_file"] = "sources" 83 | 84 | if "default_sum" not in instance: 85 | instance["default_sum"] = "md5" 86 | 87 | for host in instance["clone_hostnames"]: 88 | if host not in config_dict["clone_host_map"]: 89 | config_dict["clone_host_map"][host] = {} 90 | host_dict = config_dict["clone_host_map"][host] 91 | for prefix in instance.get("path_prefixes", ["DEFAULT"]): 92 | if prefix in host_dict: 93 | msg = "Duplicate prefix {0} for {1} hostname".format( 94 | prefix, host, 95 | ) 96 | raise RuntimeError(msg) 97 | host_dict[prefix] = instance 98 | 99 | return config_dict 100 | 101 | 102 | def download(url, filename): 103 | """ Download URL as FILENAME using curl command """ 104 | 105 | if not hasattr(download, "curl_has_retry_all_errors"): 106 | # Drop this once EL8 is not a thing to support 107 | output = check_output(["curl", "--help", "all"]) 108 | # method's static variable to avoid using 'global' 109 | download.curl_has_retry_all_errors = b"--retry-all-errors" in output 110 | 111 | command = [ 112 | "curl", 113 | "-H", "Pragma:", 114 | "-o", filename, 115 | "--location", 116 | "--connect-timeout", "60", 117 | "--retry", "3", "--retry-delay", "10", 118 | "--remote-time", 119 | "--show-error", 120 | "--fail", 121 | ] 122 | 123 | if download.curl_has_retry_all_errors: 124 | command += ["--retry-all-errors"] 125 | 126 | if call(command + [url]): 127 | raise RuntimeError("Can't download file {0}".format(filename)) 128 | 129 | 130 | def mkdir_p(path): 131 | """ mimic 'mkdir -p ' command """ 132 | try: 133 | os.makedirs(path) 134 | except OSError as err: 135 | if err.errno != errno.EEXIST: 136 | raise 137 | 138 | 139 | def download_file_and_check(url, params, distgit_config): 140 | """ Download given URL (if not yet downloaded), and try the checksum """ 141 | filename = params["filename"] 142 | sum_binary = params["hashtype"] + "sum" 143 | 144 | mkdir_p(distgit_config["sources"]) 145 | 146 | if not os.path.exists(filename): 147 | logging.info("Downloading %s", filename) 148 | download(url, filename) 149 | else: 150 | logging.info("File %s already exists", filename) 151 | 152 | sum_command = [sum_binary, filename] 153 | output = check_output(sum_command).decode("utf-8") 154 | checksum, _ = output.strip().split() 155 | if checksum != params["hash"]: 156 | raise RuntimeError("Check-sum {0} is wrong, expected: {1}".format( 157 | checksum, 158 | params["hash"], 159 | )) 160 | 161 | 162 | def _detect_clone_url(): 163 | git_config = ".git/config" 164 | if not os.path.exists(git_config): 165 | msg = "{0} not found, $PWD is not a git repository".format(git_config) 166 | raise RuntimeError(msg) 167 | 168 | git_conf_reader = configparser.ConfigParser() 169 | git_conf_reader.read(git_config) 170 | return git_conf_reader['remote "origin"']["url"] 171 | 172 | 173 | def parse_clone_url(url): 174 | """ 175 | Given Git clone url, return a named tuple with "hostname" and "path" 176 | parameters. 177 | """ 178 | Parsed = namedtuple('_ParsedUrl', ['hostname', 'path']) 179 | 180 | autoparsed = urlparse(url) 181 | if autoparsed.scheme: 182 | hostname = autoparsed.hostname or "localhost" 183 | return Parsed(hostname, autoparsed.path) 184 | 185 | if ':' in url and '@' in url: 186 | # user@hostname:/path format 187 | no_user = url.split("@", 1)[1] 188 | host, path = no_user.split(":", 1) 189 | return Parsed(host, path) 190 | 191 | # local pathname, like /home/tester/test.git 192 | return Parsed("localhost", url) 193 | 194 | 195 | def get_distgit_config(config, forked_from=None): 196 | """ 197 | Given the '.git/config' file from current directory, return the 198 | appropriate part of dist-git configuration. 199 | Returns tuple: (urlparse(clone_url), distgit_config) 200 | """ 201 | url = forked_from 202 | if not url: 203 | url = _detect_clone_url() 204 | parsed_url = parse_clone_url(url) 205 | if parsed_url.hostname is None: 206 | hostname = "localhost" 207 | else: 208 | hostname = parsed_url.hostname 209 | 210 | try: 211 | prefixes = config["clone_host_map"][hostname] 212 | except KeyError as err: 213 | raise RuntimeError(f"{hostname} (detected from clone URL) not " 214 | "found in dist-git-client configuration") from err 215 | 216 | prefix_found = None 217 | for prefix in prefixes.keys(): 218 | if not parsed_url.path.startswith(prefix): 219 | continue 220 | prefix_found = prefix 221 | 222 | if not prefix_found: 223 | if "DEFAULT" not in prefixes: 224 | raise RuntimeError("Path {0} does not match any of 'path_prefixes' " 225 | "for '{1}' hostname".format(parsed_url.path, 226 | hostname)) 227 | prefix_found = "DEFAULT" 228 | 229 | return parsed_url, prefixes[prefix_found] 230 | 231 | 232 | def get_spec(distgit_config): 233 | """ 234 | Find the specfile name inside distgit_config["specs"] directory 235 | """ 236 | spec_dir = distgit_config["specs"] 237 | specfiles = glob.glob(os.path.join(spec_dir, '*.spec')) 238 | if len(specfiles) != 1: 239 | abs_spec_dir = os.path.join(os.getcwd(), spec_dir) 240 | message = "Exactly one spec file expected in {0} directory, {1} found".format( 241 | abs_spec_dir, len(specfiles), 242 | ) 243 | raise RuntimeError(message) 244 | specfile = os.path.basename(specfiles[0]) 245 | return specfile 246 | 247 | 248 | def sources(args, config): 249 | """ 250 | Locate the sources, and download them from the appropriate dist-git 251 | lookaside cache. 252 | """ 253 | parsed_url, distgit_config = get_distgit_config(config, args.forked_from) 254 | namespace = parsed_url.path.strip('/').split('/') 255 | # drop the last {name}.git part 256 | repo_name = namespace.pop() 257 | if repo_name.endswith(".git"): 258 | repo_name = repo_name[:-4] 259 | namespace = list(reversed(namespace)) 260 | 261 | output = check_output(["git", "rev-parse", "--abbrev-ref", "HEAD"]) 262 | output = output.decode("utf-8").strip() 263 | if output == "HEAD": 264 | output = check_output(["git", "rev-parse", "HEAD"]) 265 | output = output.decode("utf-8").strip() 266 | refspec = output 267 | specfile = get_spec(distgit_config) 268 | name = specfile[:-5] 269 | sources_file = distgit_config["sources_file"].format(name=name) 270 | if not os.path.exists(sources_file): 271 | logging.info("'%s' file not found, download skipped", sources_file) 272 | return 273 | 274 | logging.info("Reading sources specification file: %s", sources_file) 275 | with open(sources_file, 'r') as sfd: 276 | while True: 277 | line = sfd.readline() 278 | if not line: 279 | break 280 | 281 | kwargs = { 282 | "name": repo_name, 283 | "refspec": refspec, 284 | "namespace": namespace, 285 | } 286 | 287 | source_spec = line.split() 288 | if not source_spec: 289 | # line full of white-spaces, skip 290 | continue 291 | 292 | if len(source_spec) == 2: 293 | # old md5/sha1 format: 0ced6f20b9fa1bea588005b5ad4b52c1 tar-1.26.tar.xz 294 | kwargs["hashtype"] = distgit_config["default_sum"].lower() 295 | kwargs["hash"] = source_spec[0] 296 | kwargs["filename"] = source_spec[1] 297 | elif len(source_spec) == 4: 298 | # SHA512 (tar-1.30.tar.xz.sig) = 299 | kwargs["hashtype"] = source_spec[0].lower() 300 | kwargs["hash"] = source_spec[3] 301 | filename = os.path.basename(source_spec[1]) 302 | kwargs["filename"] = filename.strip('()') 303 | else: 304 | msg = "Weird sources line: {0}".format(line) 305 | raise RuntimeError(msg) 306 | 307 | url_file = '/'.join([ 308 | distgit_config["lookaside_location"], 309 | distgit_config["lookaside_uri_pattern"].format(**kwargs) 310 | ]) 311 | 312 | download_file_and_check(url_file, kwargs, distgit_config) 313 | 314 | 315 | def handle_autospec(spec_abspath, spec_basename, args): 316 | """ 317 | When %auto* macros are used in SPEC_ABSPATH, expand them into a separate 318 | spec file within ARGS.OUTPUTDIR, and return the absolute filename of the 319 | specfile. When %auto* macros are not used, return SPEC_ABSPATH unchanged. 320 | """ 321 | result = spec_abspath 322 | if rpmautospec_used(spec_abspath): 323 | git_dir = check_output(["git", "rev-parse", "--git-dir"]) 324 | git_dir = git_dir.decode("utf-8").strip() 325 | if os.path.exists(os.path.join(git_dir, "shallow")): 326 | # Hack. The rpmautospec doesn't support shallow clones: 327 | # https://pagure.io/fedora-infra/rpmautospec/issue/227 328 | logging.info("rpmautospec requires full clone => --unshallow") 329 | check_call(["git", "fetch", "--unshallow"]) 330 | 331 | # Expand the %auto* macros, and create the separate spec file in the 332 | # output directory. 333 | output_spec = os.path.join(args.outputdir, spec_basename) 334 | rpmautospec_expand(spec_abspath, output_spec) 335 | result = output_spec 336 | return result 337 | 338 | 339 | def srpm(args, config): 340 | """ 341 | Using the appropriate dist-git configuration, generate source RPM 342 | file. This requires running 'def sources()' first. 343 | """ 344 | _, distgit_config = get_distgit_config(config, args.forked_from) 345 | 346 | cwd = os.getcwd() 347 | sources_dir = os.path.join(cwd, distgit_config["sources"]) 348 | specs = os.path.join(cwd, distgit_config["specs"]) 349 | spec = get_spec(distgit_config) 350 | 351 | mkdir_p(args.outputdir) 352 | 353 | spec_abspath = os.path.join(specs, spec) 354 | spec_abspath = handle_autospec(spec_abspath, spec, args) 355 | 356 | if args.mock_chroot: 357 | command = [ 358 | "mock", "--buildsrpm", 359 | "-r", args.mock_chroot, 360 | "--spec", spec_abspath, 361 | "--sources", sources_dir, 362 | "--resultdir", args.outputdir, 363 | ] 364 | else: 365 | command = [ 366 | "rpmbuild", "-bs", spec_abspath, 367 | "--define", "dist %nil", 368 | "--define", "_sourcedir {0}".format(sources_dir), 369 | "--define", "_srcrpmdir {0}".format(args.outputdir), 370 | "--define", "_disable_source_fetch 1", 371 | ] 372 | 373 | if args.dry_run or 'COPR_DISTGIT_CLIENT_DRY_RUN' in os.environ: 374 | log_cmd(command, comment="Dry run") 375 | else: 376 | check_call(command) 377 | 378 | 379 | def clone(args, config): 380 | """ 381 | Automatically clone a package from a given DistGit instance 382 | """ 383 | distgit = config["instances"][args.dist_git] 384 | parts = distgit.get("cloning_pattern_package_parts") 385 | if parts: 386 | expected = parts.split() 387 | have = args.package.split("/") 388 | if len(expected) != len(have): 389 | raise RuntimeError( 390 | "Package '{0}' has a wrong format, {1} " 391 | "slash-separated parts are expected: {2}".format( 392 | args.package, len(expected), 393 | "/".join(expected), 394 | )) 395 | 396 | clone_url = distgit["cloning_pattern"].format(package=args.package) 397 | check_call([ 398 | "git", "clone", clone_url, 399 | ]) 400 | 401 | def _get_argparser(): 402 | parser = argparse.ArgumentParser(prog="dist-git-client", 403 | description="""\ 404 | A simple, configurable python utility that is able to download sources from 405 | various dist-git instances, and generate source RPMs. 406 | The utility is able to automatically map the "origin" .git/config clone URL 407 | (or --forked-from URL, if specified) to a corresponding dist-git instance 408 | configured in /etc/dist-git-client directory. 409 | """) 410 | 411 | # main parser 412 | default_confdir = os.environ.get("COPR_DISTGIT_CLIENT_CONFDIR", 413 | "/etc/dist-git-client") 414 | parser.add_argument( 415 | "--configdir", default=default_confdir, 416 | help="Where to load configuration files from") 417 | parser.add_argument( 418 | "--loglevel", default="info", 419 | help="Python logging level, e.g. debug, info, error") 420 | parser.add_argument( 421 | "--forked-from", 422 | metavar="CLONE_URL", 423 | help=("Specify that this git clone directory is a dist-git repository " 424 | "fork. If used, the default clone url detection from the " 425 | ".git/config file is disabled and CLONE_URL is used instead. " 426 | "This specified CLONE_URL is used to detect the appropriate " 427 | "lookaside cache pattern to download the sources.")) 428 | 429 | subparsers = parser.add_subparsers( 430 | title="actions", dest="action") 431 | 432 | # sources parser 433 | subparsers.add_parser( 434 | "sources", 435 | description=( 436 | "Using the 'url' .git/config, detect where the right DistGit " 437 | "lookaside cache exists, and download the corresponding source " 438 | "files."), 439 | help="Download sources from the lookaside cache") 440 | 441 | # srpm parser 442 | srpm_parser = subparsers.add_parser( 443 | "srpm", 444 | help="Generate a source RPM", 445 | description=( 446 | "Generate a source RPM from the downloaded source files " 447 | "by 'sources' command, please run 'sources' first."), 448 | ) 449 | srpm_parser.add_argument( 450 | "--outputdir", 451 | default="/tmp", 452 | help="Where to store the resulting source RPM") 453 | srpm_parser.add_argument( 454 | "--mock-chroot", 455 | help=("Generate the SRPM in mock buildroot instead of on host. The " 456 | "argument is passed down to mock as the 'mock -r|--root' " 457 | "argument."), 458 | ) 459 | srpm_parser.add_argument( 460 | "--dry-run", action="store_true", 461 | help=("Don't produce the SRPM, just print the command which would be " 462 | "otherwise called"), 463 | ) 464 | 465 | clone_parser = subparsers.add_parser( 466 | "clone", 467 | help="Clone package from a DistGit source", 468 | ) 469 | 470 | clone_parser.add_argument( 471 | "--dist-git", 472 | default="fedora", 473 | help=("The DistGit ID as configured in /etc/dist-git-client/"), 474 | ) 475 | 476 | clone_parser.add_argument( 477 | "package", 478 | default="fedora", 479 | help=("Package name specification. For some DistGit " 480 | "instances this consists of multiple parts separated " 481 | "by slash, e.g. for '--dist-git=fedora-copr' use " 482 | "'@copr/copr-dev/copr-cli'."), 483 | ) 484 | 485 | return parser 486 | 487 | 488 | def unittests_init_git(files=None): 489 | """ 490 | Initialize .git/ directory. This method is only used for unit-testing. 491 | """ 492 | check_output(["git", "init", ".", "-b", "main"]) 493 | shutil.rmtree(".git/hooks") 494 | check_output(["git", "config", "user.email", "you@example.com"]) 495 | check_output(["git", "config", "user.name", "Your Name"]) 496 | check_output(["git", "config", "advice.detachedHead", "false"]) 497 | 498 | for filename, content in files: 499 | dirname = os.path.dirname(filename) 500 | try: 501 | os.makedirs(dirname) 502 | except OSError: 503 | pass 504 | with open(filename, "w", encoding="utf-8") as filed: 505 | filed.write(content) 506 | check_output(["git", "add", filename]) 507 | 508 | check_output(["git", "commit", "-m", "initial"]) 509 | 510 | 511 | def main(): 512 | """ The entrypoint for the whole logic """ 513 | args = _get_argparser().parse_args() 514 | logging.basicConfig( 515 | level=getattr(logging, args.loglevel.upper()), 516 | format="%(levelname)s: %(message)s", 517 | ) 518 | config = _load_config(args.configdir) 519 | 520 | try: 521 | if args.action == "srpm": 522 | srpm(args, config) 523 | elif args.action == "clone": 524 | clone(args, config) 525 | else: 526 | sources(args, config) 527 | except RuntimeError as err: 528 | logging.error("%s", err) 529 | sys.exit(1) 530 | -------------------------------------------------------------------------------- /dist-git-client/etc/default.ini: -------------------------------------------------------------------------------- 1 | # *Match* the given repository clone_url's with an appropriate DistGit 2 | # configuration (== identify the corresponding lookaside cache location). 3 | # 4 | # Any *.ini file in /etc/copr-distgit-client directory is parsed. The format is 5 | # just INI (python ConfigParser format). The section names denote the DistGit 6 | # instances' IDs. Each section can have the following options: 7 | # 8 | # clone_hostnames: List[string] 9 | # List of possible hostnames to consider when matching this configuration. 10 | # E.g. "example.com" matches both "git://example.com/foo" and 11 | # "https://example.com/foo.git" URLs. 12 | # 13 | # path_prefixes: List[string] (optional) 14 | # List of possible path prefixes to consider. Complements the 15 | # "clone_hostnames" option above. When specified, **both** 16 | # "clone_hostnames" and 'path_prefixes' must match to use the 17 | # corresponding configuration section. E.g. "/foo/bar" prefix matches 18 | # the "https://example.com/foo/bar/rpms/component.git" clone_url. 19 | # When not specified, **any** prefix is accepted on matched hostname. 20 | # 21 | # sources_file: String (optional, pathname) 22 | # Expected 'sources' file location for this DistGit instance (relative to 23 | # the git root directory). Defaault = './sources'. 24 | # 25 | # specs: String (optional, pathname) 26 | # Expected spec file directory, relative to the git root directory. 27 | # Default = '.' (spec files stored directly in the git root). 28 | # 29 | # sources: String (optional, pathname) 30 | # Where the source files (referenced by 'sources_file') should be 31 | # downloaded. Default = '.' (git root directory). 32 | # 33 | # default_sum: String (capital, optional) 34 | # Up2date 'sources' files explicitly denote the checksum type used for 35 | # given files using lines like "SHA512 () = ". But still, 36 | # even the "old" sources file format is supported with lines like 37 | # " ". This option is to define what sum type is expected 38 | # in this DistGit instance (Default = MD5). 39 | # 40 | # lookaside_location: String (hostname) 41 | # Url of the storage where to download the sources from. 42 | # 43 | # lookaside_uri_pattern: String 44 | # Relative path on the 'lookaside_location' where the files should be 45 | # found, given the info parsed from 'sources' file. Possible fields are 46 | # "name" (component), "filename", "hashtype" (e.g. 'md5'), "hash" 47 | # (checksum), "namespace" (array, prefix before the component name). 48 | 49 | [fedora] 50 | clone_hostnames = 51 | pkgs.fedoraproject.org 52 | src.fedoraproject.org 53 | lookaside_location = https://src.fedoraproject.org 54 | lookaside_uri_pattern = repo/pkgs/rpms/{name}/{filename}/{hashtype}/{hash}/{filename} 55 | cloning_pattern = https://src.fedoraproject.org/rpms/{package}.git 56 | 57 | [centos] 58 | clone_hostnames = git.centos.org 59 | lookaside_location = https://git.centos.org 60 | sources_file = .{name}.metadata 61 | specs = SPECS 62 | sources = SOURCES 63 | default_sum = SHA1 64 | lookaside_uri_pattern = sources/{name}/{refspec}/{hash} 65 | cloning_pattern = https://git.centos.org/rpms/{package}.git 66 | 67 | [fedora-copr] 68 | clone_hostnames = copr-dist-git.fedorainfracloud.org 69 | lookaside_location = https://copr-dist-git.fedorainfracloud.org 70 | lookaside_uri_pattern = repo/pkgs/{namespace[1]}/{namespace[0]}/{name}/{filename}/{hashtype}/{hash}/{filename} 71 | cloning_pattern_package_parts = owner_name project_name package_name 72 | cloning_pattern=https://copr-dist-git.fedorainfracloud.org/git/{package} 73 | 74 | [fedora-copr-dev] 75 | clone_hostnames = copr-dist-git-dev.fedorainfracloud.org 76 | lookaside_location = https://copr-dist-git-dev.fedorainfracloud.org 77 | lookaside_uri_pattern = repo/pkgs/{namespace[1]}/{namespace[0]}/{name}/{filename}/{hashtype}/{hash}/{filename} 78 | cloning_pattern_package_parts = owner_name project_name package_name 79 | cloning_pattern=https://copr-dist-git-dev.fedorainfracloud.org/git/{package} 80 | 81 | [centos-stream] 82 | clone_hostnames = gitlab.com 83 | path_prefixes = /redhat/centos-stream/rpms 84 | lookaside_location = https://sources.stream.centos.org 85 | lookaside_uri_pattern = sources/rpms/{name}/{filename}/{hashtype}/{hash}/{filename} 86 | cloning_pattern = https://gitlab.com/redhat/centos-stream/rpms/{package}.git 87 | -------------------------------------------------------------------------------- /dist-git-client/run_tests.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | set -e 4 | 5 | args=() 6 | 7 | coverage=( --cov-report term-missing --cov bin --cov dist_git_client ) 8 | for arg; do 9 | case $arg in 10 | --no-coverage) coverage=() ;; 11 | *) args+=( "$arg" ) ;; 12 | esac 13 | done 14 | 15 | abspath=$(readlink -f .) 16 | export PYTHONPATH="${PYTHONPATH+$PYTHONPATH:}$abspath" 17 | export PATH=$(readlink -f bin):$PATH 18 | "${PYTHON:-python3}" -m pytest -s tests "${coverage[@]}" "${args[@]}" 19 | -------------------------------------------------------------------------------- /dist-git-client/tags: -------------------------------------------------------------------------------- 1 | !_TAG_EXTRA_DESCRIPTION anonymous /Include tags for non-named objects like lambda/ 2 | !_TAG_EXTRA_DESCRIPTION fileScope /Include tags of file scope/ 3 | !_TAG_EXTRA_DESCRIPTION pseudo /Include pseudo tags/ 4 | !_TAG_EXTRA_DESCRIPTION qualified /Include an extra class-qualified tag entry for each tag/ 5 | !_TAG_EXTRA_DESCRIPTION subparser /Include tags generated by subparsers/ 6 | !_TAG_FIELD_DESCRIPTION access /Access (or export) of class members/ 7 | !_TAG_FIELD_DESCRIPTION epoch /the last modified time of the input file (only for F\/file kind tag)/ 8 | !_TAG_FIELD_DESCRIPTION file /File-restricted scoping/ 9 | !_TAG_FIELD_DESCRIPTION inherits /Inheritance information/ 10 | !_TAG_FIELD_DESCRIPTION input /input file/ 11 | !_TAG_FIELD_DESCRIPTION name /tag name/ 12 | !_TAG_FIELD_DESCRIPTION pattern /pattern/ 13 | !_TAG_FIELD_DESCRIPTION signature /Signature of routine (e.g. prototype or parameter list)/ 14 | !_TAG_FIELD_DESCRIPTION typeref /Type and name of a variable or typedef/ 15 | !_TAG_FIELD_DESCRIPTION!Python nameref /the original name for the tag/ 16 | !_TAG_FILE_FORMAT 2 /extended format; --format=1 will not append ;" to lines/ 17 | !_TAG_FILE_SORTED 1 /0=unsorted, 1=sorted, 2=foldcase/ 18 | !_TAG_KIND_DESCRIPTION!Iniconf k,key /keys/ 19 | !_TAG_KIND_DESCRIPTION!Iniconf s,section /sections/ 20 | !_TAG_KIND_DESCRIPTION!Markdown S,subsection /level 2 sections/ 21 | !_TAG_KIND_DESCRIPTION!Markdown T,l4subsection /level 4 sections/ 22 | !_TAG_KIND_DESCRIPTION!Markdown c,chapter /chapters/ 23 | !_TAG_KIND_DESCRIPTION!Markdown n,footnote /footnotes/ 24 | !_TAG_KIND_DESCRIPTION!Markdown s,section /sections/ 25 | !_TAG_KIND_DESCRIPTION!Markdown t,subsubsection /level 3 sections/ 26 | !_TAG_KIND_DESCRIPTION!Markdown u,l5subsection /level 5 sections/ 27 | !_TAG_KIND_DESCRIPTION!Python I,namespace /name referring a module defined in other file/ 28 | !_TAG_KIND_DESCRIPTION!Python Y,unknown /name referring a class\/variable\/function\/module defined in other module/ 29 | !_TAG_KIND_DESCRIPTION!Python c,class /classes/ 30 | !_TAG_KIND_DESCRIPTION!Python f,function /functions/ 31 | !_TAG_KIND_DESCRIPTION!Python i,module /modules/ 32 | !_TAG_KIND_DESCRIPTION!Python m,member /class members/ 33 | !_TAG_KIND_DESCRIPTION!Python v,variable /variables/ 34 | !_TAG_KIND_DESCRIPTION!RpmSpec g,global /global macros/ 35 | !_TAG_KIND_DESCRIPTION!RpmSpec m,macro /macros/ 36 | !_TAG_KIND_DESCRIPTION!RpmSpec p,package /packages/ 37 | !_TAG_KIND_DESCRIPTION!RpmSpec p,patch /patch files/ 38 | !_TAG_KIND_DESCRIPTION!RpmSpec t,tag /tags/ 39 | !_TAG_KIND_DESCRIPTION!Sh a,alias /aliases/ 40 | !_TAG_KIND_DESCRIPTION!Sh f,function /functions/ 41 | !_TAG_KIND_DESCRIPTION!Sh h,heredoc /label for here document/ 42 | !_TAG_KIND_DESCRIPTION!Sh s,script /script files/ 43 | !_TAG_OUTPUT_EXCMD mixed /number, pattern, mixed, or combineV2/ 44 | !_TAG_OUTPUT_FILESEP slash /slash or backslash/ 45 | !_TAG_OUTPUT_MODE u-ctags /u-ctags or e-ctags/ 46 | !_TAG_OUTPUT_VERSION 0.0 /current.age/ 47 | !_TAG_PARSER_VERSION!Iniconf 0.0 /current.age/ 48 | !_TAG_PARSER_VERSION!Markdown 0.0 /current.age/ 49 | !_TAG_PARSER_VERSION!Python 0.0 /current.age/ 50 | !_TAG_PARSER_VERSION!RpmSpec 0.0 /current.age/ 51 | !_TAG_PARSER_VERSION!Sh 0.0 /current.age/ 52 | !_TAG_PATTERN_LENGTH_LIMIT 96 /0 for no limit/ 53 | !_TAG_PROC_CWD /home/praiskup/rh/projects/copr/copr/worktree/praiskup/dist-git-client/dist-git-client/ // 54 | !_TAG_PROGRAM_AUTHOR Universal Ctags Team // 55 | !_TAG_PROGRAM_NAME Universal Ctags /Derived from Exuberant Ctags/ 56 | !_TAG_PROGRAM_URL https://ctags.io/ /official site/ 57 | !_TAG_PROGRAM_VERSION 6.0.0 // 58 | !_TAG_ROLE_DESCRIPTION!Python!module imported /imported modules/ 59 | !_TAG_ROLE_DESCRIPTION!Python!module indirectlyImported /module imported in alternative name/ 60 | !_TAG_ROLE_DESCRIPTION!Python!module namespace /namespace from where classes\/variables\/functions are imported/ 61 | !_TAG_ROLE_DESCRIPTION!Python!unknown imported /imported from the other module/ 62 | !_TAG_ROLE_DESCRIPTION!Python!unknown indirectlyImported /classes\/variables\/functions\/modules imported in alternative name/ 63 | !_TAG_ROLE_DESCRIPTION!RpmSpec!macro undef /undefined/ 64 | !_TAG_ROLE_DESCRIPTION!RpmSpec!patch decl /declared for applying later/ 65 | !_TAG_ROLE_DESCRIPTION!Sh!heredoc endmarker /end marker/ 66 | !_TAG_ROLE_DESCRIPTION!Sh!script loaded /loaded/ 67 | BuildArch dist-git-client.spec /^BuildArch: noarch$/;" t 68 | BuildRequires dist-git-client.spec /^BuildRequires: \/usr\/bin\/argparse-manpage$/;" t 69 | BuildRequires dist-git-client.spec /^BuildRequires: python-rpm-macros$/;" t 70 | BuildRequires dist-git-client.spec /^BuildRequires: python3-rpmautospec$/;" t 71 | License dist-git-client.spec /^License: GPL-2.0-or-later$/;" t 72 | Name dist-git-client.spec /^Name: dist-git-client$/;" t 73 | Release dist-git-client.spec /^Release: 1%{?dist}$/;" t 74 | Requires dist-git-client.spec /^Requires: %{_bindir}\/git$/;" t 75 | Requires dist-git-client.spec /^Requires: curl$/;" t 76 | Requires dist-git-client.spec /^Requires: python3-rpmautospec$/;" t 77 | Requires dist-git-client.spec /^Requires: python3-six$/;" t 78 | Source0 dist-git-client.spec /^Source0: %name-%version.tar.gz$/;" t 79 | Summary dist-git-client.spec /^Summary: Get sources for RPM builds from DistGit repositories$/;" t 80 | TestDistGitDownload tests/test_dist_git_client.py /^class TestDistGitDownload(object):$/;" c inherits:object access:public 81 | TestDistGitDownload.args tests/test_dist_git_client.py /^ args = None$/;" v class:TestDistGitDownload access:public 82 | TestDistGitDownload.config tests/test_dist_git_client.py /^ config = None$/;" v class:TestDistGitDownload access:public 83 | TestDistGitDownload.config_dir tests/test_dist_git_client.py /^ config_dir = None$/;" v class:TestDistGitDownload access:public 84 | TestDistGitDownload.setup_method tests/test_dist_git_client.py /^ def setup_method(self, method):$/;" m class:TestDistGitDownload access:public signature:(self, method) 85 | TestDistGitDownload.setup_method._Args tests/test_dist_git_client.py /^ class _Args:$/;" c member:TestDistGitDownload.setup_method file: inherits: access:private 86 | TestDistGitDownload.setup_method._Args.dry_run tests/test_dist_git_client.py /^ dry_run = False$/;" v class:TestDistGitDownload.setup_method._Args access:public 87 | TestDistGitDownload.setup_method._Args.forked_from tests/test_dist_git_client.py /^ forked_from = None$/;" v class:TestDistGitDownload.setup_method._Args access:public 88 | TestDistGitDownload.teardown_method tests/test_dist_git_client.py /^ def teardown_method(self, method):$/;" m class:TestDistGitDownload access:public signature:(self, method) 89 | TestDistGitDownload.test_centos tests/test_dist_git_client.py /^ def test_centos(self, download):$/;" m class:TestDistGitDownload access:public signature:(self, download) 90 | TestDistGitDownload.test_centos_download tests/test_dist_git_client.py /^ def test_centos_download(self, patched_check_call):$/;" m class:TestDistGitDownload access:public signature:(self, patched_check_call) 91 | TestDistGitDownload.test_copr_distgit tests/test_dist_git_client.py /^ def test_copr_distgit(self, download):$/;" m class:TestDistGitDownload access:public signature:(self, download) 92 | TestDistGitDownload.test_duplicate_prefix tests/test_dist_git_client.py /^ def test_duplicate_prefix(self):$/;" m class:TestDistGitDownload access:public signature:(self) 93 | TestDistGitDownload.test_fedora_new tests/test_dist_git_client.py /^ def test_fedora_new(self, download, base):$/;" m class:TestDistGitDownload access:public signature:(self, download, base) 94 | TestDistGitDownload.test_fedora_old tests/test_dist_git_client.py /^ def test_fedora_old(self, download, base):$/;" m class:TestDistGitDownload access:public signature:(self, download, base) 95 | TestDistGitDownload.test_load_prefix tests/test_dist_git_client.py /^ def test_load_prefix(self):$/;" m class:TestDistGitDownload access:public signature:(self) 96 | TestDistGitDownload.test_load_prefix_fail tests/test_dist_git_client.py /^ def test_load_prefix_fail(self):$/;" m class:TestDistGitDownload access:public signature:(self) 97 | TestDistGitDownload.test_no_git_config tests/test_dist_git_client.py /^ def test_no_git_config(self):$/;" m class:TestDistGitDownload access:public signature:(self) 98 | TestDistGitDownload.test_no_spec tests/test_dist_git_client.py /^ def test_no_spec(self):$/;" m class:TestDistGitDownload access:public signature:(self) 99 | TestDistGitDownload.workdir tests/test_dist_git_client.py /^ workdir = None$/;" v class:TestDistGitDownload access:public 100 | URL dist-git-client.spec /^URL: https:\/\/github.com\/release-engineering\/dist-git.git$/;" t 101 | Version dist-git-client.spec /^Version: 1.1$/;" t 102 | _Args tests/test_dist_git_client.py /^ class _Args:$/;" c member:TestDistGitDownload.setup_method file: inherits: access:private 103 | _detect_clone_url dist_git_client.py /^def _detect_clone_url():$/;" f access:protected signature:() 104 | _get_argparser dist_git_client.py /^def _get_argparser():$/;" f access:protected signature:() 105 | _load_config dist_git_client.py /^def _load_config(directory):$/;" f access:protected signature:(directory) 106 | args tests/test_dist_git_client.py /^ args = None$/;" v class:TestDistGitDownload access:public 107 | call dist_git_client.py /^def call(cmd, comment="Calling"):$/;" f access:public signature:(cmd, comment="Calling") 108 | centos etc/default.ini /^[centos]$/;" s 109 | centos-stream etc/default.ini /^[centos-stream]$/;" s 110 | check_call dist_git_client.py /^def check_call(cmd, comment="Checked call"):$/;" f access:public signature:(cmd, comment="Checked call") 111 | check_output dist_git_client.py /^def check_output(cmd, comment="Reading stdout from command"):$/;" f access:public signature:(cmd, comment="Reading stdout from command") 112 | clone dist_git_client.py /^def clone(args, config):$/;" f access:public signature:(args, config) 113 | clone_hostnames etc/default.ini /^clone_hostnames = copr-dist-git-dev.fedorainfracloud.org$/;" k section:fedora-copr-dev 114 | clone_hostnames etc/default.ini /^clone_hostnames = copr-dist-git.fedorainfracloud.org$/;" k section:fedora-copr 115 | clone_hostnames etc/default.ini /^clone_hostnames = git.centos.org$/;" k section:centos 116 | clone_hostnames etc/default.ini /^clone_hostnames = gitlab.com$/;" k section:centos-stream 117 | clone_hostnames etc/default.ini /^clone_hostnames =$/;" k section:fedora 118 | cloning_pattern etc/default.ini /^cloning_pattern = https:\/\/git.centos.org\/rpms\/{package}.git$/;" k section:centos 119 | cloning_pattern etc/default.ini /^cloning_pattern = https:\/\/gitlab.com\/redhat\/centos-stream\/rpms\/{package}.git$/;" k section:centos-stream 120 | cloning_pattern etc/default.ini /^cloning_pattern = https:\/\/src.fedoraproject.org\/rpms\/{package}.git$/;" k section:fedora 121 | cloning_pattern etc/default.ini /^cloning_pattern=https:\/\/copr-dist-git-dev.fedorainfracloud.org\/git\/{package}$/;" k section:fedora-copr-dev 122 | cloning_pattern etc/default.ini /^cloning_pattern=https:\/\/copr-dist-git.fedorainfracloud.org\/git\/{package}$/;" k section:fedora-copr 123 | cloning_pattern_package_parts etc/default.ini /^cloning_pattern_package_parts = owner_name project_name package_name$/;" k section:fedora-copr 124 | cloning_pattern_package_parts etc/default.ini /^cloning_pattern_package_parts = owner_name project_name package_name$/;" k section:fedora-copr-dev 125 | config tests/test_dist_git_client.py /^ config = None$/;" v class:TestDistGitDownload access:public 126 | config_dir tests/test_dist_git_client.py /^ config_dir = None$/;" v class:TestDistGitDownload access:public 127 | default_sum etc/default.ini /^default_sum = SHA1$/;" k section:centos 128 | dist-git-client dist-git-client.spec /^Name: dist-git-client$/;" p 129 | download dist_git_client.py /^def download(url, filename):$/;" f access:public signature:(url, filename) 130 | download_file_and_check dist_git_client.py /^def download_file_and_check(url, params, distgit_config):$/;" f access:public signature:(url, params, distgit_config) 131 | dry_run tests/test_dist_git_client.py /^ dry_run = False$/;" v class:TestDistGitDownload.setup_method._Args access:public 132 | fedora etc/default.ini /^[fedora]$/;" s 133 | fedora-copr etc/default.ini /^[fedora-copr]$/;" s 134 | fedora-copr-dev etc/default.ini /^[fedora-copr-dev]$/;" s 135 | forked_from tests/test_dist_git_client.py /^ forked_from = None$/;" v class:TestDistGitDownload.setup_method._Args access:public 136 | get_distgit_config dist_git_client.py /^def get_distgit_config(config, forked_from=None):$/;" f access:public signature:(config, forked_from=None) 137 | get_spec dist_git_client.py /^def get_spec(distgit_config):$/;" f access:public signature:(distgit_config) 138 | git_origin_url tests/test_dist_git_client.py /^def git_origin_url(url):$/;" f access:public signature:(url) 139 | handle_autospec dist_git_client.py /^def handle_autospec(spec_abspath, spec_basename, args):$/;" f access:public signature:(spec_abspath, spec_basename, args) 140 | log_cmd dist_git_client.py /^def log_cmd(command, comment="Running command"):$/;" f access:public signature:(command, comment="Running command") 141 | lookaside_location etc/default.ini /^lookaside_location = https:\/\/copr-dist-git-dev.fedorainfracloud.org$/;" k section:fedora-copr-dev 142 | lookaside_location etc/default.ini /^lookaside_location = https:\/\/copr-dist-git.fedorainfracloud.org$/;" k section:fedora-copr 143 | lookaside_location etc/default.ini /^lookaside_location = https:\/\/git.centos.org$/;" k section:centos 144 | lookaside_location etc/default.ini /^lookaside_location = https:\/\/sources.stream.centos.org$/;" k section:centos-stream 145 | lookaside_location etc/default.ini /^lookaside_location = https:\/\/src.fedoraproject.org$/;" k section:fedora 146 | lookaside_uri_pattern etc/default.ini /^lookaside_uri_pattern = repo\/pkgs\/rpms\/{name}\/{filename}\/{hashtype}\/{hash}\/{filename}$/;" k section:fedora 147 | lookaside_uri_pattern etc/default.ini /^lookaside_uri_pattern = repo\/pkgs\/{namespace[1]}\/{namespace[0]}\/{name}\/{filename}\/{hashtyp/;" k section:fedora-copr 148 | lookaside_uri_pattern etc/default.ini /^lookaside_uri_pattern = repo\/pkgs\/{namespace[1]}\/{namespace[0]}\/{name}\/{filename}\/{hashtyp/;" k section:fedora-copr-dev 149 | lookaside_uri_pattern etc/default.ini /^lookaside_uri_pattern = sources\/rpms\/{name}\/{filename}\/{hashtype}\/{hash}\/{filename}$/;" k section:centos-stream 150 | lookaside_uri_pattern etc/default.ini /^lookaside_uri_pattern = sources\/{name}\/{refspec}\/{hash}$/;" k section:centos 151 | main dist_git_client.py /^def main():$/;" f access:public signature:() 152 | mkdir_p dist_git_client.py /^def mkdir_p(path):$/;" f access:public signature:(path) 153 | path_prefixes etc/default.ini /^path_prefixes = \/redhat\/centos-stream\/rpms$/;" k section:centos-stream 154 | pytest cache directory .pytest_cache/README.md /^# pytest cache directory #$/;" c 155 | rpmautospec_expand dist_git_client.py /^ process_distgit as rpmautospec_expand,$/;" Y access:public nameref:unknown:process_distgit 156 | rpmautospec_used dist_git_client.py /^ specfile_uses_rpmautospec as rpmautospec_used,$/;" Y access:public nameref:unknown:specfile_uses_rpmautospec 157 | rpmautospec_used dist_git_client.py /^ rpmautospec_used = lambda _: False$/;" f access:public signature:(_) 158 | setup_method tests/test_dist_git_client.py /^ def setup_method(self, method):$/;" m class:TestDistGitDownload access:public signature:(self, method) 159 | sources dist_git_client.py /^def sources(args, config):$/;" f access:public signature:(args, config) 160 | sources etc/default.ini /^sources = SOURCES$/;" k section:centos 161 | sources_file etc/default.ini /^sources_file = .{name}.metadata$/;" k section:centos 162 | specs etc/default.ini /^specs = SPECS$/;" k section:centos 163 | srpm dist_git_client.py /^def srpm(args, config):$/;" f access:public signature:(args, config) 164 | teardown_method tests/test_dist_git_client.py /^ def teardown_method(self, method):$/;" m class:TestDistGitDownload access:public signature:(self, method) 165 | test_centos tests/test_dist_git_client.py /^ def test_centos(self, download):$/;" m class:TestDistGitDownload access:public signature:(self, download) 166 | test_centos_download tests/test_dist_git_client.py /^ def test_centos_download(self, patched_check_call):$/;" m class:TestDistGitDownload access:public signature:(self, patched_check_call) 167 | test_copr_distgit tests/test_dist_git_client.py /^ def test_copr_distgit(self, download):$/;" m class:TestDistGitDownload access:public signature:(self, download) 168 | test_duplicate_prefix tests/test_dist_git_client.py /^ def test_duplicate_prefix(self):$/;" m class:TestDistGitDownload access:public signature:(self) 169 | test_fedora_new tests/test_dist_git_client.py /^ def test_fedora_new(self, download, base):$/;" m class:TestDistGitDownload access:public signature:(self, download, base) 170 | test_fedora_old tests/test_dist_git_client.py /^ def test_fedora_old(self, download, base):$/;" m class:TestDistGitDownload access:public signature:(self, download, base) 171 | test_load_prefix tests/test_dist_git_client.py /^ def test_load_prefix(self):$/;" m class:TestDistGitDownload access:public signature:(self) 172 | test_load_prefix_fail tests/test_dist_git_client.py /^ def test_load_prefix_fail(self):$/;" m class:TestDistGitDownload access:public signature:(self) 173 | test_no_git_config tests/test_dist_git_client.py /^ def test_no_git_config(self):$/;" m class:TestDistGitDownload access:public signature:(self) 174 | test_no_spec tests/test_dist_git_client.py /^ def test_no_spec(self):$/;" m class:TestDistGitDownload access:public signature:(self) 175 | tests_init_git dist_git_client.py /^def tests_init_git(files=None):$/;" f access:public signature:(files=None) 176 | workdir tests/test_dist_git_client.py /^ workdir = None$/;" v class:TestDistGitDownload access:public 177 | -------------------------------------------------------------------------------- /dist-git-client/tests/test_dist_git_client.py: -------------------------------------------------------------------------------- 1 | """ 2 | dist-git-client testsuite 3 | """ 4 | 5 | import os 6 | import shutil 7 | import tempfile 8 | 9 | import pytest 10 | 11 | try: 12 | from unittest import mock 13 | except ImportError: 14 | import mock 15 | 16 | from dist_git_client import (sources, srpm, _load_config, check_output, 17 | _detect_clone_url, get_distgit_config, unittests_init_git, 18 | ) 19 | 20 | # pylint: disable=useless-object-inheritance 21 | 22 | def git_origin_url(url): 23 | """ setup .git/config with core.origin.url == URL """ 24 | with open(".git/config", "a+") as gcf: 25 | gcf.write('[remote "origin"]\n') 26 | gcf.write('url = {0}\n'.format(url)) 27 | 28 | 29 | class TestDistGitDownload(object): 30 | """ Test the 'sources()' method """ 31 | config = None 32 | args = None 33 | workdir = None 34 | config_dir = None 35 | 36 | def setup_method(self, method): 37 | _unused_but_needed_for_el6 = (method) 38 | testdir = os.path.dirname(__file__) 39 | projdir = os.path.dirname(testdir) 40 | self.config_dir = os.path.join(projdir, 'etc') 41 | self.config = _load_config(self.config_dir) 42 | class _Args: 43 | # pylint: disable=too-few-public-methods 44 | dry_run = False 45 | forked_from = None 46 | self.args = _Args() 47 | self.workdir = tempfile.mkdtemp(prefix="dist-git-client-test-") 48 | os.chdir(self.workdir) 49 | 50 | def teardown_method(self, method): 51 | _unused_but_needed_for_el6 = (method) 52 | shutil.rmtree(self.workdir) 53 | 54 | 55 | @mock.patch('dist_git_client.download_file_and_check') 56 | def test_copr_distgit(self, download): 57 | unittests_init_git([ 58 | # .spec in .git, in Copr it is possible 59 | ("test.spec", ""), 60 | ("sources", "2102fd0602de72e58765adcbf92349d8 retrace-server-git-955.3e4742a.tar.gz\n"), 61 | ]) 62 | git_origin_url("https://copr-dist-git.fedorainfracloud.org/git/@abrt/retrace-server-devel/retrace-server.git") 63 | sources(self.args, self.config) 64 | assert len(download.call_args_list) == 1 65 | assert download.call_args_list[0][0][0] == ( 66 | "https://copr-dist-git.fedorainfracloud.org/repo/pkgs/" 67 | "@abrt/retrace-server-devel/retrace-server/retrace-server-git-955.3e4742a.tar.gz/" 68 | "md5/2102fd0602de72e58765adcbf92349d8/retrace-server-git-955.3e4742a.tar.gz" 69 | ) 70 | 71 | @pytest.mark.parametrize('base', ["tar", "tar/"]) 72 | @mock.patch('dist_git_client.download_file_and_check') 73 | def test_fedora_old(self, download, base): 74 | """ 75 | Old sources format + ssh clone 76 | """ 77 | unittests_init_git([ 78 | ("tar.spec", ""), 79 | ("sources", "0ced6f20b9fa1bea588005b5ad4b52c1 tar-1.26.tar.xz\n"), 80 | ]) 81 | git_origin_url("ssh://praiskup@pkgs.fedoraproject.org/rpms/" + base) 82 | sources(self.args, self.config) 83 | assert len(download.call_args_list) == 1 84 | assert download.call_args_list[0][0][0] == ( 85 | "https://src.fedoraproject.org/repo/pkgs/rpms/" 86 | "tar/tar-1.26.tar.xz/md5/0ced6f20b9fa1bea588005b5ad4b52c1/tar-1.26.tar.xz" 87 | ) 88 | 89 | @pytest.mark.parametrize('base', ["tar.git", "tar.git/"]) 90 | @mock.patch('dist_git_client.download_file_and_check') 91 | def test_fedora_new(self, download, base): 92 | """ 93 | New sources format + anonymous clone 94 | """ 95 | sha512 = ( 96 | "1bd13854009b6ee08958481738e6bf661e40216a2befe461d06b4b350eb882e43" 97 | "1b3a4eeea7ca1d35d37102df76194c9d933df2b18b3c5401350e9fc17017750" 98 | ) 99 | unittests_init_git([ 100 | ("tar.spec", ""), 101 | ("sources", "SHA512 (tar-1.32.tar.xz) = {0}\n".format(sha512)), 102 | ]) 103 | git_origin_url("https://src.fedoraproject.org/rpms/" + base) 104 | sources(self.args, self.config) 105 | assert len(download.call_args_list) == 1 106 | url = ( 107 | "https://src.fedoraproject.org/repo/pkgs/rpms/" 108 | "tar/tar-1.32.tar.xz/sha512/{sha512}/tar-1.32.tar.xz" 109 | ).format(sha512=sha512) 110 | assert download.call_args_list[0][0][0] == url 111 | 112 | @mock.patch('dist_git_client.download_file_and_check') 113 | def test_centos(self, download): 114 | """ 115 | Anonymous centos clone 116 | """ 117 | unittests_init_git([ 118 | ("SPECS/centpkg-minimal.spec", ""), 119 | (".centpkg-minimal.metadata", "cf9ce8d900768ed352a6f19a2857e64403643545 SOURCES/centpkg-minimal.tar.gz\n"), 120 | ]) 121 | git_origin_url("https://git.centos.org/rpms/centpkg-minimal.git") 122 | sources(self.args, self.config) 123 | assert len(download.call_args_list) == 1 124 | assert download.call_args_list[0][0][0] == ( 125 | "https://git.centos.org/sources/centpkg-minimal/main/" 126 | "cf9ce8d900768ed352a6f19a2857e64403643545" 127 | ) 128 | assert download.call_args_list[0][0][2]["sources"] == "SOURCES" 129 | assert download.call_args_list[0][0][1]["hashtype"] == "sha1" 130 | 131 | oldref = check_output(["git", "rev-parse", "HEAD"]).decode("utf-8") 132 | oldref = oldref.strip() 133 | 134 | # create new commit, and checkout back (so --show-current is not set) 135 | check_output(["git", "commit", "--allow-empty", "-m", "empty"]) 136 | check_output(["git", "checkout", "-q", oldref]) 137 | 138 | sources(self.args, self.config) 139 | assert download.call_args_list[1][0][0] == ( 140 | "https://git.centos.org/sources/centpkg-minimal/{0}/" 141 | "cf9ce8d900768ed352a6f19a2857e64403643545" 142 | ).format(oldref) 143 | 144 | 145 | @mock.patch("dist_git_client.subprocess.check_call") 146 | def test_centos_download(self, patched_check_call): 147 | unittests_init_git([ 148 | ("SPECS/centpkg-minimal.spec", ""), 149 | (".centpkg-minimal.metadata", "cf9ce8d900768ed352a6f19a2857e64403643545 SOURCES/centpkg-minimal.tar.gz\n"), 150 | ]) 151 | git_origin_url("https://git.centos.org/rpms/centpkg-minimal.git") 152 | setattr(self.args, "outputdir", os.path.join(self.workdir, "result")) 153 | setattr(self.args, "mock_chroot", None) 154 | srpm(self.args, self.config) 155 | assert patched_check_call.call_args_list[0][0][0] == [ 156 | 'rpmbuild', '-bs', 157 | os.path.join(self.workdir, "SPECS", "centpkg-minimal.spec"), 158 | '--define', 'dist %nil', 159 | '--define', '_sourcedir ' + self.workdir + '/SOURCES', 160 | '--define', '_srcrpmdir ' + self.workdir + '/result', 161 | '--define', '_disable_source_fetch 1', 162 | ] 163 | 164 | def test_duplicate_prefix(self): 165 | modified_dir = os.path.join(self.workdir, "config") 166 | shutil.copytree(self.config_dir, modified_dir) 167 | modified_file = os.path.join(modified_dir, "default.ini") 168 | with open(modified_file, "a+") as fmodify: 169 | fmodify.write( 170 | "\n\n[hack]\n" 171 | "clone_hostnames = gitlab.com\n" 172 | "path_prefixes = /redhat/centos-stream/rpms\n" 173 | ) 174 | with pytest.raises(RuntimeError) as err: 175 | _load_config(modified_dir) 176 | assert "Duplicate prefix /redhat" in str(err) 177 | 178 | def test_no_git_config(self): 179 | with pytest.raises(RuntimeError) as err: 180 | _detect_clone_url() 181 | assert "is not a git" in str(err) 182 | 183 | def test_load_prefix(self): 184 | prefixed_url = "git://gitlab.com/redhat/centos-stream/rpms/test.git" 185 | _, config = get_distgit_config(self.config, forked_from=prefixed_url) 186 | assert config["lookaside_location"] == "https://sources.stream.centos.org" 187 | 188 | def test_load_prefix_fail(self): 189 | prefixed_url = "git://gitlab.com/non-existent/centos-stream/rpms/test.git" 190 | with pytest.raises(RuntimeError) as err: 191 | get_distgit_config(self.config, forked_from=prefixed_url) 192 | msg = "Path /non-existent/centos-stream/rpms/test.git does not " + \ 193 | "match any of 'path_prefixes' for 'gitlab.com' hostname" 194 | assert msg in str(err) 195 | 196 | def test_no_spec(self): 197 | unittests_init_git([ 198 | ("sources", "0ced6f20b9fa1bea588005b5ad4b52c1 tar-1.26.tar.xz\n"), 199 | ]) 200 | git_origin_url("ssh://praiskup@pkgs.fedoraproject.org/rpms/tar") 201 | with pytest.raises(RuntimeError) as err: 202 | sources(self.args, self.config) 203 | strings = [ 204 | "directory, 0 found", 205 | "Exactly one spec file expected in", 206 | ] 207 | for string in strings: 208 | assert string in str(err) 209 | -------------------------------------------------------------------------------- /dist-git-client/tests/test_url_parser.py: -------------------------------------------------------------------------------- 1 | """ test clone url parser """ 2 | 3 | from dist_git_client import parse_clone_url 4 | 5 | 6 | def _checker(url, hostname, path): 7 | parsed = parse_clone_url(url) 8 | assert parsed.hostname == hostname 9 | assert parsed.path == path 10 | 11 | 12 | def test_parse_clone_urls(): 13 | """ Basic clone url formats """ 14 | _checker("git@github.com:example/example-project.git", 15 | "github.com", "example/example-project.git") 16 | _checker("https://github.com/example/example-project.git", 17 | "github.com", "/example/example-project.git") 18 | _checker("https://github.com/example/example-project", 19 | "github.com", "/example/example-project") 20 | _checker("ssh://jdoe@pkgs.fedoraproject.org/rpms/example.git", 21 | "pkgs.fedoraproject.org", "/rpms/example.git") 22 | _checker("https://copr-dist-git.fedorainfracloud.org/git" 23 | "/@abrt/retrace-server-devel/retrace-server.git", 24 | "copr-dist-git.fedorainfracloud.org", 25 | "/git/@abrt/retrace-server-devel/retrace-server.git") 26 | _checker("file:///home/foo.git", 27 | "localhost", "/home/foo.git") 28 | _checker("/home/foo.git", 29 | "localhost", "/home/foo.git") 30 | -------------------------------------------------------------------------------- /dist-git/LICENSE: -------------------------------------------------------------------------------- 1 | This file contains three licenses: 2 | - MIT License 3 | - GNU GENERAL PUBLIC LICENSE Version 1 4 | 5 | -------------------------------------------------------------------------------- 6 | 7 | MIT License: 8 | 9 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 10 | 11 | The above copyright notice and this permission notice (including the next paragraph) shall be included in all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 14 | 15 | -------------------------------------------------------------------------------- 16 | 17 | 18 | GNU GENERAL PUBLIC LICENSE 19 | Version 1, February 1989 20 | 21 | Copyright (C) 1989 Free Software Foundation, Inc. 22 | 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA 23 | 24 | Everyone is permitted to copy and distribute verbatim copies 25 | of this license document, but changing it is not allowed. 26 | 27 | Preamble 28 | 29 | The license agreements of most software companies try to keep users 30 | at the mercy of those companies. By contrast, our General Public 31 | License is intended to guarantee your freedom to share and change free 32 | software--to make sure the software is free for all its users. The 33 | General Public License applies to the Free Software Foundation's 34 | software and to any other program whose authors commit to using it. 35 | You can use it for your programs, too. 36 | 37 | When we speak of free software, we are referring to freedom, not 38 | price. Specifically, the General Public License is designed to make 39 | sure that you have the freedom to give away or sell copies of free 40 | software, that you receive source code or can get it if you want it, 41 | that you can change the software or use pieces of it in new free 42 | programs; and that you know you can do these things. 43 | 44 | To protect your rights, we need to make restrictions that forbid 45 | anyone to deny you these rights or to ask you to surrender the rights. 46 | These restrictions translate to certain responsibilities for you if you 47 | distribute copies of the software, or if you modify it. 48 | 49 | For example, if you distribute copies of a such a program, whether 50 | gratis or for a fee, you must give the recipients all the rights that 51 | you have. You must make sure that they, too, receive or can get the 52 | source code. And you must tell them their rights. 53 | 54 | We protect your rights with two steps: (1) copyright the software, and 55 | (2) offer you this license which gives you legal permission to copy, 56 | distribute and/or modify the software. 57 | 58 | Also, for each author's protection and ours, we want to make certain 59 | that everyone understands that there is no warranty for this free 60 | software. If the software is modified by someone else and passed on, we 61 | want its recipients to know that what they have is not the original, so 62 | that any problems introduced by others will not reflect on the original 63 | authors' reputations. 64 | 65 | The precise terms and conditions for copying, distribution and 66 | modification follow. 67 | 68 | GNU GENERAL PUBLIC LICENSE 69 | TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 70 | 71 | 0. This License Agreement applies to any program or other work which 72 | contains a notice placed by the copyright holder saying it may be 73 | distributed under the terms of this General Public License. The 74 | "Program", below, refers to any such program or work, and a "work based 75 | on the Program" means either the Program or any work containing the 76 | Program or a portion of it, either verbatim or with modifications. Each 77 | licensee is addressed as "you". 78 | 79 | 1. You may copy and distribute verbatim copies of the Program's source 80 | code as you receive it, in any medium, provided that you conspicuously and 81 | appropriately publish on each copy an appropriate copyright notice and 82 | disclaimer of warranty; keep intact all the notices that refer to this 83 | General Public License and to the absence of any warranty; and give any 84 | other recipients of the Program a copy of this General Public License 85 | along with the Program. You may charge a fee for the physical act of 86 | transferring a copy. 87 | 88 | 2. You may modify your copy or copies of the Program or any portion of 89 | it, and copy and distribute such modifications under the terms of Paragraph 90 | 1 above, provided that you also do the following: 91 | 92 | a) cause the modified files to carry prominent notices stating that 93 | you changed the files and the date of any change; and 94 | 95 | b) cause the whole of any work that you distribute or publish, that 96 | in whole or in part contains the Program or any part thereof, either 97 | with or without modifications, to be licensed at no charge to all 98 | third parties under the terms of this General Public License (except 99 | that you may choose to grant warranty protection to some or all 100 | third parties, at your option). 101 | 102 | c) If the modified program normally reads commands interactively when 103 | run, you must cause it, when started running for such interactive use 104 | in the simplest and most usual way, to print or display an 105 | announcement including an appropriate copyright notice and a notice 106 | that there is no warranty (or else, saying that you provide a 107 | warranty) and that users may redistribute the program under these 108 | conditions, and telling the user how to view a copy of this General 109 | Public License. 110 | 111 | d) You may charge a fee for the physical act of transferring a 112 | copy, and you may at your option offer warranty protection in 113 | exchange for a fee. 114 | 115 | Mere aggregation of another independent work with the Program (or its 116 | derivative) on a volume of a storage or distribution medium does not bring 117 | the other work under the scope of these terms. 118 | 119 | 3. You may copy and distribute the Program (or a portion or derivative of 120 | it, under Paragraph 2) in object code or executable form under the terms of 121 | Paragraphs 1 and 2 above provided that you also do one of the following: 122 | 123 | a) accompany it with the complete corresponding machine-readable 124 | source code, which must be distributed under the terms of 125 | Paragraphs 1 and 2 above; or, 126 | 127 | b) accompany it with a written offer, valid for at least three 128 | years, to give any third party free (except for a nominal charge 129 | for the cost of distribution) a complete machine-readable copy of the 130 | corresponding source code, to be distributed under the terms of 131 | Paragraphs 1 and 2 above; or, 132 | 133 | c) accompany it with the information you received as to where the 134 | corresponding source code may be obtained. (This alternative is 135 | allowed only for noncommercial distribution and only if you 136 | received the program in object code or executable form alone.) 137 | 138 | Source code for a work means the preferred form of the work for making 139 | modifications to it. For an executable file, complete source code means 140 | all the source code for all modules it contains; but, as a special 141 | exception, it need not include source code for modules which are standard 142 | libraries that accompany the operating system on which the executable 143 | file runs, or for standard header files or definitions files that 144 | accompany that operating system. 145 | 146 | 4. You may not copy, modify, sublicense, distribute or transfer the 147 | Program except as expressly provided under this General Public License. 148 | Any attempt otherwise to copy, modify, sublicense, distribute or transfer 149 | the Program is void, and will automatically terminate your rights to use 150 | the Program under this License. However, parties who have received 151 | copies, or rights to use copies, from you under this General Public 152 | License will not have their licenses terminated so long as such parties 153 | remain in full compliance. 154 | 155 | 5. By copying, distributing or modifying the Program (or any work based 156 | on the Program) you indicate your acceptance of this license to do so, 157 | and all its terms and conditions. 158 | 159 | 6. Each time you redistribute the Program (or any work based on the 160 | Program), the recipient automatically receives a license from the original 161 | licensor to copy, distribute or modify the Program subject to these 162 | terms and conditions. You may not impose any further restrictions on the 163 | recipients' exercise of the rights granted herein. 164 | 165 | 7. The Free Software Foundation may publish revised and/or new versions 166 | of the General Public License from time to time. Such new versions will 167 | be similar in spirit to the present version, but may differ in detail to 168 | address new problems or concerns. 169 | 170 | Each version is given a distinguishing version number. If the Program 171 | specifies a version number of the license which applies to it and "any 172 | later version", you have the option of following the terms and conditions 173 | either of that version or of any later version published by the Free 174 | Software Foundation. If the Program does not specify a version number of 175 | the license, you may choose any version ever published by the Free Software 176 | Foundation. 177 | 178 | 8. If you wish to incorporate parts of the Program into other free 179 | programs whose distribution conditions are different, write to the author 180 | to ask for permission. For software which is copyrighted by the Free 181 | Software Foundation, write to the Free Software Foundation; we sometimes 182 | make exceptions for this. Our decision will be guided by the two goals 183 | of preserving the free status of all derivatives of our free software and 184 | of promoting the sharing and reuse of software generally. 185 | 186 | NO WARRANTY 187 | 188 | 9. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY 189 | FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN 190 | OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES 191 | PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED 192 | OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 193 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS 194 | TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE 195 | PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, 196 | REPAIR OR CORRECTION. 197 | 198 | 10. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 199 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR 200 | REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, 201 | INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING 202 | OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED 203 | TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY 204 | YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER 205 | PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE 206 | POSSIBILITY OF SUCH DAMAGES. 207 | 208 | END OF TERMS AND CONDITIONS 209 | 210 | Appendix: How to Apply These Terms to Your New Programs 211 | 212 | If you develop a new program, and you want it to be of the greatest 213 | possible use to humanity, the best way to achieve this is to make it 214 | free software which everyone can redistribute and change under these 215 | terms. 216 | 217 | To do so, attach the following notices to the program. It is safest to 218 | attach them to the start of each source file to most effectively convey 219 | the exclusion of warranty; and each file should have at least the 220 | "copyright" line and a pointer to where the full notice is found. 221 | 222 | 223 | Copyright (C) 19yy 224 | 225 | This program is free software; you can redistribute it and/or modify 226 | it under the terms of the GNU General Public License as published by 227 | the Free Software Foundation; either version 1, or (at your option) 228 | any later version. 229 | 230 | This program is distributed in the hope that it will be useful, 231 | but WITHOUT ANY WARRANTY; without even the implied warranty of 232 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 233 | GNU General Public License for more details. 234 | 235 | You should have received a copy of the GNU General Public License 236 | along with this program; if not, write to the Free Software 237 | Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301 USA 238 | 239 | 240 | Also add information on how to contact you by electronic and paper mail. 241 | 242 | If the program is interactive, make it output a short notice like this 243 | when it starts in an interactive mode: 244 | 245 | Gnomovision version 69, Copyright (C) 19xx name of author 246 | Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. 247 | This is free software, and you are welcome to redistribute it 248 | under certain conditions; type `show c' for details. 249 | 250 | The hypothetical commands `show w' and `show c' should show the 251 | appropriate parts of the General Public License. Of course, the 252 | commands you use may be called something other than `show w' and `show 253 | c'; they could even be mouse-clicks or menu items--whatever suits your 254 | program. 255 | 256 | You should also get your employer (if you work as a programmer) or your 257 | school, if any, to sign a "copyright disclaimer" for the program, if 258 | necessary. Here a sample; alter the names: 259 | 260 | Yoyodyne, Inc., hereby disclaims all copyright interest in the 261 | program `Gnomovision' (a program to direct compilers to make passes 262 | at assemblers) written by James Hacker. 263 | 264 | , 1 April 1989 265 | Ty Coon, President of Vice 266 | 267 | That's all there is to it! 268 | 269 | 270 | -------------------------------------------------------------------------------- /dist-git/README.md: -------------------------------------------------------------------------------- 1 | DistGit 2 | ======= 3 | 4 | DistGit (Distribution Git) is Git with additional data storage. It is designed to hold content of source rpms and consists of these three main components: 5 | 6 | 1. Git repositories 7 | 2. Lookaside cache to store source tarballs 8 | 3. Scripts to manage both 9 | 10 | Read here for information about the most recent release: https://github.com/release-engineering/dist-git/wiki 11 | 12 | How Does It Work 13 | ---------------- 14 | 15 | RPM source package typically contains a spec file and the sources (upstream tarball + additional patches). Source tarballs, being binary and potentially large, are not very well suited to be placed in a Git repository. On each their update, Git would produce a huge, meaningless diff. That's why DistGit was introduced as it employs an efficient lookaside cache where the tarballs can be stored. The Git repo itself can then be left to do what it does best: keep track of changes on the spec file, downstream patches, and an additional text file called `sources` that contains link to the source tarball in the lookaside cache. 16 | 17 | ![storage](/dist-git/images/storage.png) 18 | 19 | Video Tutorial 20 | -------------- 21 | 22 | [![DistGit video tutorial](/dist-git/images/tutorial.png)](https://www.youtube.com/watch?v=VsnJymZRQOM "DistGit video tutorial") 23 | 24 | User Guide 25 | ---------- 26 | 27 | #### 1. Build and Install the Package: 28 | 29 | The project is prepared to be built as an RPM package. You can easily build it on [Fedora](https://getfedora.org/) or [CentOS](https://www.centos.org/) with EPEL7 enabled using a tool called [tito](https://github.com/rpm-software-management/tito). 30 | To build the current release, use the following command in the repo directory: 31 | 32 | ``` 33 | $ tito build --rpm 34 | ``` 35 | 36 | Install the resulting RPM package: 37 | 38 | ``` 39 | # tito build --rpm -i 40 | ``` 41 | 42 | #### 2. Configuration: 43 | 44 | Enable the lookaside cache by using and modifying the example httpd config: 45 | 46 | ``` 47 | # cd /etc/httpd/conf.d/dist-git/ 48 | # cp lookaside-upload.conf.example lookaside-upload.conf 49 | # vim lookaside-upload.conf 50 | ``` 51 | 52 | Lookaside Cache uses https communication and client authenticates with ssl client certificate. The Dist Git service provider needs to issue the client certificate for every user. 53 | 54 | #### 3. Users and Groups: 55 | 56 | All DistGit users need to: 57 | 58 | 1. have an ssh server access with private key authentication 59 | 2. be in a *packager* group on the server 60 | 3. be provided with an ssl client certificate to authenticate with the lookaside cache 61 | 62 | #### 4. Install DistGit Web Interface: 63 | 64 | Install Cgit, the web interface for Git: 65 | 66 | ``` 67 | # dnf install cgit 68 | ``` 69 | 70 | And point it to the DistGit repositories: 71 | 72 | ``` 73 | echo "scan-path=/var/lib/dist-git/git/" >> /etc/cgitrc 74 | ``` 75 | 76 | It is useful to comment out `cache-size` entry in /etc/cgitrc (or set it to zero) to always get up-to-date repository state at each page refresh. 77 | 78 | The web interface will be available on address like `http://your-server/cgit`. 79 | 80 | #### 5. Systemd Services: 81 | 82 | ``` 83 | # systemctl start sshd 84 | # systemctl start httpd 85 | # systemctl start dist-git.socket 86 | ``` 87 | 88 | #### 6. DistGit client tools: 89 | 90 | To interact with DistGit server, you can use use [rpkg](https://pagure.io/rpkg-util) or [fedpkg](https://pagure.io/fedpkg) command-line tools. 91 | 92 | #### 7. Deployment 93 | 94 | You can see examples of Ansible deployment scripts in 95 | [Fedora Infastructure dist-git role] and [Copr dist-git role]. 96 | 97 | 98 | ### Related 99 | 100 | * [Source-git](https://packit.dev/docs/source-git/) - project started in 2020. Intended as layer on top of dist-git. 101 | 102 | Developer Guide 103 | --------------- 104 | 105 | #### Unit tests 106 | 107 | ``` 108 | $ pytest -v . 109 | ``` 110 | 111 | #### Integration tests 112 | 113 | Please, see `beaker-tests/README.md`. 114 | 115 | LICENSE 116 | ------- 117 | 118 | Whole project use MIT license. File upload.cgi uses GPLv1. 119 | 120 | 121 | [Fedora Infastructure dist-git role]: https://pagure.io/fedora-infra/ansible/blob/main/f/roles/distgit 122 | [Copr dist-git role]: https://pagure.io/fedora-infra/ansible/blob/main/f/roles/copr/dist_git 123 | -------------------------------------------------------------------------------- /dist-git/Vagrantfile: -------------------------------------------------------------------------------- 1 | # -*- mode: ruby -*- 2 | # vi: set ft=ruby : 3 | 4 | Vagrant.configure(2) do |config| 5 | 6 | # increase memory because default 512MB 7 | # doesn't seem to be enough for dnf these days 8 | config.vm.provider :libvirt do |v| 9 | v.memory = 1024 10 | end 11 | 12 | ### DistGit Fedora ################################################### 13 | # we would like to say dist-git in |...| but that is invalid syntax 14 | config.vm.define "dist-git" do |distgit| 15 | distgit.vm.box = "fedora/32-cloud-base" 16 | 17 | distgit.vm.synced_folder ".", "/vagrant", type: "rsync" 18 | 19 | distgit.vm.provision "shell", 20 | inline: "echo 'nameserver 8.8.8.8' >> /etc/resolv.conf" 21 | 22 | # Copy config files 23 | distgit.vm.provision "shell", 24 | inline: "rm -rf /tmp/pkgs-files", 25 | run: "always" 26 | 27 | distgit.vm.provision "file", 28 | source: "./beaker-tests/pkgs-files", destination: "/tmp/", 29 | run: "always" 30 | 31 | # update the system & install the packages 32 | distgit.vm.provision "shell", 33 | inline: "dnf clean all && dnf -y update || true" # || true cause dnf might return non-zero status (probly delta rpm rebuilt failed) 34 | 35 | distgit.vm.provision "shell", 36 | inline: "dnf install -y tito wget" 37 | 38 | distgit.vm.provision "shell", 39 | inline: "dnf builddep -y /vagrant/dist-git.spec", 40 | run: "always" 41 | 42 | distgit.vm.provision "shell", 43 | inline: "rm -rf /tmp/tito/noarch", 44 | run: "always" 45 | 46 | distgit.vm.provision "shell", 47 | privileged: false, 48 | inline: "cd /vagrant/ && tito build --test --rpm", 49 | run: "always" 50 | 51 | distgit.vm.provision "shell", 52 | inline: "dnf install -y /tmp/tito/noarch/*.rpm", 53 | run: "always" 54 | 55 | # setup config files 56 | distgit.vm.provision "shell", 57 | inline: "mv /tmp/pkgs-files/ssl.conf /etc/httpd/conf.d/ssl.conf && restorecon -R /etc/httpd/conf.d/ssl.conf", 58 | run: "always" 59 | 60 | distgit.vm.provision "shell", 61 | inline: "mv /tmp/pkgs-files/lookaside-upload.conf /etc/httpd/conf.d/dist-git/ && restorecon -R /etc/httpd/conf.d/dist-git/", 62 | run: "always" 63 | 64 | # setup test user 65 | distgit.vm.provision "shell", 66 | inline: "useradd clime -G packager" 67 | 68 | distgit.vm.provision "shell", 69 | inline: "echo 'clime ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers" 70 | 71 | # start services 72 | distgit.vm.provision "shell", 73 | inline: "systemctl enable dist-git.socket && systemctl restart dist-git.socket", 74 | run: "always" 75 | 76 | distgit.vm.provision "shell", 77 | inline: "systemctl enable httpd && systemctl restart httpd", 78 | run: "always" 79 | 80 | end 81 | 82 | ### DistGit CentOS 7 ################################################### 83 | config.vm.define "dist-git-centos-7" do |distgit| 84 | distgit.vm.box = "centos/7" 85 | 86 | distgit.vm.synced_folder ".", "/vagrant", type: "rsync" 87 | 88 | distgit.vm.provision "shell", 89 | inline: "echo 'nameserver 8.8.8.8' >> /etc/resolv.conf" 90 | 91 | # Copy config files 92 | distgit.vm.provision "shell", 93 | inline: "rm -rf /tmp/pkgs-files", 94 | run: "always" 95 | 96 | distgit.vm.provision "file", 97 | source: "./beaker-tests/pkgs-files", destination: "/tmp/", 98 | run: "always" 99 | 100 | # enable epel7 repo 101 | distgit.vm.provision "shell", 102 | inline: "yum install -y epel-release", 103 | run: "always" 104 | 105 | # update the system & install the packages 106 | distgit.vm.provision "shell", 107 | inline: "yum clean all && yum -y update || true" 108 | 109 | distgit.vm.provision "shell", 110 | inline: "yum install -y tito wget net-tools" 111 | 112 | distgit.vm.provision "shell", 113 | inline: "yum-builddep -y /vagrant/dist-git.spec", 114 | run: "always" 115 | 116 | distgit.vm.provision "shell", 117 | inline: "rm -rf /tmp/tito/noarch", 118 | run: "always" 119 | 120 | distgit.vm.provision "shell", 121 | privileged: false, 122 | inline: "cd /vagrant/ && tito build --test --rpm", 123 | run: "always" 124 | 125 | distgit.vm.provision "shell", 126 | inline: "yum install -y /tmp/tito/noarch/*.rpm || true", 127 | run: "always" 128 | 129 | distgit.vm.provision "shell", 130 | inline: "yum install -y python-grokmirror", 131 | run: "always" 132 | 133 | # setup config files 134 | distgit.vm.provision "shell", 135 | inline: "mv /tmp/pkgs-files/ssl.conf /etc/httpd/conf.d/ssl.conf && restorecon -R /etc/httpd/conf.d/ssl.conf", 136 | run: "always" 137 | 138 | distgit.vm.provision "shell", 139 | inline: "mv /tmp/pkgs-files/lookaside-upload.conf /etc/httpd/conf.d/dist-git/ && restorecon -R /etc/httpd/conf.d/dist-git/", 140 | run: "always" 141 | 142 | # setup test user 143 | distgit.vm.provision "shell", 144 | inline: "useradd clime -G packager" 145 | 146 | distgit.vm.provision "shell", 147 | inline: "echo 'clime ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers" 148 | 149 | # start services 150 | distgit.vm.provision "shell", 151 | inline: "systemctl enable dist-git.socket && systemctl restart dist-git.socket", 152 | run: "always" 153 | 154 | distgit.vm.provision "shell", 155 | inline: "systemctl enable httpd && systemctl restart httpd", 156 | run: "always" 157 | end 158 | 159 | ### DistGit CentOS 8 ################################################### 160 | config.vm.define "dist-git-centos-8" do |distgit| 161 | distgit.vm.box = "centos/8" 162 | 163 | distgit.vm.synced_folder ".", "/vagrant", type: "rsync" 164 | 165 | distgit.vm.provision "shell", 166 | inline: "echo 'nameserver 8.8.8.8' >> /etc/resolv.conf" 167 | 168 | # Copy config files 169 | distgit.vm.provision "shell", 170 | inline: "rm -rf /tmp/pkgs-files", 171 | run: "always" 172 | 173 | distgit.vm.provision "file", 174 | source: "./beaker-tests/pkgs-files", destination: "/tmp/", 175 | run: "always" 176 | 177 | # enable epel8 repo 178 | distgit.vm.provision "shell", 179 | inline: "yum install -y epel-release", 180 | run: "always" 181 | 182 | # update the system & install the packages 183 | distgit.vm.provision "shell", 184 | inline: "yum clean all && yum -y update || true" 185 | 186 | distgit.vm.provision "shell", 187 | inline: "yum install -y wget net-tools" 188 | 189 | distgit.vm.provision "shell", 190 | inline: "yum-builddep -y /vagrant/dist-git.spec", 191 | run: "always" 192 | 193 | distgit.vm.provision "shell", 194 | inline: "curl https://copr.fedorainfracloud.org/coprs/clime/rpkg-util-v2/repo/epel-8/clime-rpkg-util-v2-epel-8.repo > /etc/yum.repos.d/clime-rpkg-util-epel-8.repo", 195 | run: "always" 196 | 197 | distgit.vm.provision "shell", 198 | inline: "yum install -y rpkg", 199 | run: "always" 200 | 201 | # enable auto-packing for rpkg 202 | distgit.vm.provision "shell", 203 | inline: "sed -i 's/^auto_pack = False$/auto_pack = True/' /etc/rpkg.conf", 204 | run: "always" 205 | 206 | distgit.vm.provision "shell", 207 | inline: "rm -rf /tmp/rpkg", 208 | run: "always" 209 | 210 | # build by using auto-packing 211 | distgit.vm.provision "shell", 212 | privileged: false, 213 | inline: "cd /vagrant/ && rpkg srpm && rpkg local --outdir /tmp/rpkg/dist-git-*/", 214 | run: "always" 215 | 216 | distgit.vm.provision "shell", 217 | inline: "yum install -y /tmp/rpkg/dist-git-*/noarch/*.rpm || true", 218 | run: "always" 219 | 220 | # https://bugzilla.redhat.com/show_bug.cgi?id=1833810 221 | #distgit.vm.provision "shell", 222 | # inline: "yum install -y python3-grokmirror", 223 | # run: "always" 224 | 225 | # setup config files 226 | distgit.vm.provision "shell", 227 | inline: "mv /tmp/pkgs-files/ssl.conf /etc/httpd/conf.d/ssl.conf && restorecon -R /etc/httpd/conf.d/ssl.conf", 228 | run: "always" 229 | 230 | distgit.vm.provision "shell", 231 | inline: "mv /tmp/pkgs-files/lookaside-upload.conf /etc/httpd/conf.d/dist-git/ && restorecon -R /etc/httpd/conf.d/dist-git/", 232 | run: "always" 233 | 234 | # setup test user 235 | distgit.vm.provision "shell", 236 | inline: "useradd clime -G packager" 237 | 238 | distgit.vm.provision "shell", 239 | inline: "echo 'clime ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers" 240 | 241 | # start services 242 | distgit.vm.provision "shell", 243 | inline: "systemctl enable dist-git.socket && systemctl restart dist-git.socket", 244 | run: "always" 245 | 246 | distgit.vm.provision "shell", 247 | inline: "systemctl enable httpd && systemctl restart httpd", 248 | run: "always" 249 | end 250 | end 251 | -------------------------------------------------------------------------------- /dist-git/beaker-tests/Makefile: -------------------------------------------------------------------------------- 1 | # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 2 | # 3 | # Makefile of /tools/dist-git/Regression/ 4 | # Description: Test dist-git functionality. 5 | # Author: clime 6 | # 7 | # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 8 | # 9 | # Copyright (c) 2016 Red Hat, Inc. 10 | # 11 | # This program is free software: you can redistribute it and/or 12 | # modify it under the terms of the GNU General Public License as 13 | # published by the Free Software Foundation, either version 2 of 14 | # the License, or (at your option) any later version. 15 | # 16 | # This program is distributed in the hope that it will be 17 | # useful, but WITHOUT ANY WARRANTY; without even the implied 18 | # warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR 19 | # PURPOSE. See the GNU General Public License for more details. 20 | # 21 | # You should have received a copy of the GNU General Public License 22 | # along with this program. If not, see http://www.gnu.org/licenses/. 23 | # 24 | # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 25 | 26 | export TEST=/tools/dist-git/Regression/ 27 | export TESTVERSION=1.0 28 | 29 | BUILT_FILES= 30 | 31 | FILES=$(METADATA) run.sh Makefile PURPOSE pkgs-files files setup.sh tests 32 | 33 | .PHONY: all install download clean 34 | 35 | run: $(FILES) build 36 | ./run.sh 37 | 38 | build: $(BUILT_FILES) 39 | test -x run.sh || chmod a+x run.sh 40 | 41 | clean: 42 | rm -f *~ $(BUILT_FILES) 43 | 44 | 45 | include /usr/share/rhts/lib/rhts-make.include 46 | 47 | $(METADATA): Makefile 48 | @echo "Owner: clime " > $(METADATA) 49 | @echo "Name: $(TEST)" >> $(METADATA) 50 | @echo "TestVersion: $(TESTVERSION)" >> $(METADATA) 51 | @echo "Path: $(TEST_DIR)" >> $(METADATA) 52 | @echo "Description: Test dist-git functionality." >> $(METADATA) 53 | @echo "Type: Regression" >> $(METADATA) 54 | @echo "TestTime: 2h" >> $(METADATA) 55 | @echo "RunFor: dist-git" >> $(METADATA) 56 | @echo "Requires: dist-git" >> $(METADATA) 57 | @echo "Priority: Normal" >> $(METADATA) 58 | @echo "License: GPLv2+" >> $(METADATA) 59 | @echo "Confidential: no" >> $(METADATA) 60 | @echo "Destructive: no" >> $(METADATA) 61 | @echo "Releases: -RHEL4 -RHELClient5 -RHELServer5" >> $(METADATA) 62 | 63 | rhts-lint $(METADATA) 64 | -------------------------------------------------------------------------------- /dist-git/beaker-tests/PURPOSE: -------------------------------------------------------------------------------- 1 | PURPOSE of /tools/dist-git/Regression/ 2 | Description: Test dist-git functionality. 3 | Author: clime 4 | -------------------------------------------------------------------------------- /dist-git/beaker-tests/README.md: -------------------------------------------------------------------------------- 1 | How to run tests 2 | ================ 3 | 4 | The following command will setup your system, setup a Fedora virtual machine and run tests from the `./tests/` subdirectory. You need to be root and the command will modify your system, i.e. install some packages, overwrite `/etc/rpkg/*.conf` configs and `/etc/rpkg.conf`, write an entry into `/etc/hosts`, and generates root ssh public key if not generated yet. 5 | 6 | ``` 7 | # ./run.sh 8 | ``` 9 | 10 | The setup part for most of the host system modifications and creation of the virtual machine is done by `./setup.sh` script that is invoked from `run.sh` and is the part that requires root privileges. Use `cat setup.sh` to see what the script does. You can choose not to run the script during `./run.sh` execution if you have run it before (either through `./run.sh` or `./setup.sh` directly). To do this, pass `-x` switch to `./run.sh`: 11 | 12 | ``` 13 | # ./run.sh -x 14 | ``` 15 | 16 | To run the tests for CentOS-7 (with epel-7 repos enabled) instead of Fedora, you can use: 17 | 18 | ``` 19 | # DISTGIT_FLAVOR=dist-git-centos-7 ./run.sh 20 | ``` 21 | 22 | This will spawn a new CentOS-7/epel-7 virtual machine. 23 | 24 | To do the same for CentOS-8 (with epel-8 repos enabled), you can use: 25 | 26 | ``` 27 | # DISTGIT_FLAVOR=dist-git-centos-8 ./run.sh 28 | ``` 29 | 30 | Between each run, you need to remove `pkgs.example.org` record from `/root/.ssh/known_hosts` and you should also clean-up `/etc/hosts` from stale `pkgs.example.org` records. 31 | 32 | Once the virtual machine you want to test exists and once it is correctly pointed to by `pkgs.example.org` IP record in `/etc/hosts`, you can again run just `./run.sh -x` to skip the `./setup.sh` invocation. 33 | 34 | At this point, you can also run just a specific test by switching into `./tests/` subdirectory and invoking `./run.sh` there. Alternatively, you can pass `-r ` together with `-x` switch to the master `./run.sh` script (the one next to this README): 35 | 36 | ``` 37 | # ./run.sh -x -r fedpkg-test 38 | ``` 39 | 40 | If you want to look inside the created virtual machines, switch to the git top-level directory and run `vagrant ssh ` command there as root, e.g.: 41 | 42 | ``` 43 | # vagrant ssh dist-git 44 | ``` 45 | 46 | to switch to the Fedora virtual machine. 47 | 48 | Or 49 | 50 | ``` 51 | # vagrant ssh dist-git-centos-8 52 | ``` 53 | 54 | to switch to CentOS-8/epel-8 virtual machine. 55 | 56 | *NOTES:* 57 | 58 | If you encounter vagrant error like: `Call to virDomainCreateWithFlags failed: Cannot access storage file '/root/.local/share/libvirt/images/dist-git_dist-git.img' (as uid:107, gid:107): Permission denied`, add the following into `/etc/libvirt/qemu.conf`. 59 | 60 | ``` 61 | user=root 62 | group=root 63 | ``` 64 | -------------------------------------------------------------------------------- /dist-git/beaker-tests/data/prunerepo-1.1-1.fc23.src.rpm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/release-engineering/dist-git/851f4d9500c148a81229fcba237c0ac687a4de40/dist-git/beaker-tests/data/prunerepo-1.1-1.fc23.src.rpm -------------------------------------------------------------------------------- /dist-git/beaker-tests/files/etc/rpkg.conf: -------------------------------------------------------------------------------- 1 | [rpkg] 2 | preprocess_spec = True 3 | 4 | # auto-packing is deprecated: 5 | auto_pack = False 6 | 7 | base_output_path = /tmp/rpkg 8 | 9 | # module_name = @group/project 10 | 11 | [git] 12 | lookaside = http://pkgs.example.org/repo/pkgs/%(ns1)s/%(pkg)s/%(filename)s/%(hashtype)s/%(hash)s/%(filename)s 13 | lookaside_cgi = http://pkgs.example.org/repo/pkgs/upload.cgi 14 | gitbaseurl = ssh://clime@pkgs.example.org/var/lib/dist-git/git/%(module)s 15 | anongiturl = git://pkgs.example.org/%(module)s 16 | -------------------------------------------------------------------------------- /dist-git/beaker-tests/files/etc/rpkg/fedpkg.conf: -------------------------------------------------------------------------------- 1 | [fedpkg] 2 | lookaside = http://pkgs.example.org/repo/pkgs 3 | lookasidehash = sha512 4 | lookaside_cgi = http://pkgs.example.org/repo/pkgs/upload.cgi 5 | gitbaseurl = ssh://clime@pkgs.example.org/var/lib/dist-git/git/%(module)s 6 | anongiturl = git://pkgs.example.org/%(module)s 7 | tracbaseurl = https://clime:%(password)s@fedorahosted.org/rel-eng/login/xmlrpc 8 | branchre = f\d$|f\d\d$|el\d$|olpc\d$|master$ 9 | kojiconfig = /etc/koji.conf 10 | build_client = koji 11 | clone_config = 12 | bz.default-tracker bugzilla.redhat.com 13 | bz.default-product Fedora 14 | bz.default-version rawhide 15 | bz.default-component %(repo)s 16 | sendemail.to %(repo)s-owner@fedoraproject.org 17 | clone_config_rpms = 18 | bz.default-tracker bugzilla.redhat.com 19 | bz.default-product Fedora 20 | bz.default-version rawhide 21 | bz.default-component %(repo)s 22 | sendemail.to %(repo)s-owner@fedoraproject.org 23 | clone_config_modules = 24 | bz.default-tracker bugzilla.redhat.com 25 | bz.default-product Fedora Modules 26 | bz.default-version rawhide 27 | bz.default-component %(repo)s 28 | sendemail.to module-%(repo)s-owner@fedoraproject.org 29 | clone_config_container = 30 | bz.default-tracker bugzilla.redhat.com 31 | bz.default-product Fedora Container Images 32 | bz.default-version rawhide 33 | bz.default-component %(repo)s 34 | sendemail.to container-%(repo)s-owner@fedoraproject.org 35 | distgit_namespaced = True 36 | distgit_namespaces = rpms container modules flatpaks 37 | lookaside_namespaced = True 38 | kerberos_realms = FEDORAPROJECT.ORG 39 | oidc_id_provider = https://id.fedoraproject.org/openidc/ 40 | oidc_client_id = fedpkg 41 | oidc_client_secret = notsecret 42 | oidc_scopes = openid,https://id.fedoraproject.org/scope/groups,https://mbs.fedoraproject.org/oidc/submit-build,https://src.fedoraproject.org/push 43 | git_excludes = 44 | i386/ 45 | i686/ 46 | x86_64/ 47 | ppc/ 48 | ppc64/ 49 | ia64/ 50 | mips/ 51 | arm/ 52 | noarch/ 53 | /*.src.rpm 54 | /build*.log 55 | /.build-*.log 56 | results_*/ 57 | clog 58 | 59 | [fedpkg.bodhi] 60 | # This is for the bodhi-client 2.x, that do not require an option to switch to 61 | # different instance. Instead, --staging is available to switch to the stage 62 | # bodhi, and production is used without providing --staging. 63 | staging = False 64 | releases_service = https://bodhi.fedoraproject.org/releases/%(release)s 65 | 66 | [fedpkg.mbs] 67 | auth_method = oidc 68 | api_url = https://mbs.fedoraproject.org/module-build-service/1/ 69 | oidc_id_provider = https://id.fedoraproject.org/openidc/ 70 | oidc_client_id = mbs-authorizer 71 | oidc_client_secret = notsecret 72 | oidc_scopes = openid,https://id.fedoraproject.org/scope/groups,https://mbs.fedoraproject.org/oidc/submit-build 73 | 74 | [fedpkg.bugzilla] 75 | url = https://bugzilla.redhat.com/ 76 | 77 | [fedpkg.pagure] 78 | url = https://pagure.io/ 79 | token = 80 | 81 | [fedpkg.pdc] 82 | url = https://pdc.fedoraproject.org/ 83 | 84 | [fedpkg.greenwave] 85 | url = https://greenwave.fedoraproject.org/ 86 | 87 | [fedpkg.distgit] 88 | apibaseurl = https://src.fedoraproject.org 89 | token = 90 | -------------------------------------------------------------------------------- /dist-git/beaker-tests/files/etc/rpkg/rhpkg.conf: -------------------------------------------------------------------------------- 1 | [rhpkg] 2 | lookaside = http://pkgs.example.org/repo/pkgs 3 | lookasidehash = md5 4 | lookaside_cgi = http://pkgs.example.org/repo/pkgs/upload.cgi 5 | gitbaseurl = ssh://clime@pkgs.example.org/var/lib/dist-git/git/%(module)s 6 | anongiturl = git://pkgs.example.org/%(module)s 7 | 8 | branchre = rhel 9 | kojiprofile = brew 10 | build_client = brew 11 | kerberos_realms = REDHAT.COM 12 | distgit_namespaced = True 13 | lookaside_namespaced = True 14 | 15 | [rhpkg.dist-git] 16 | commit_url = http://pkgs.devel.redhat.com/cgit/%(module)s/commit/?id=%(commit_hash)s 17 | 18 | [rhpkg.covscan] 19 | executable = covscan 20 | 21 | [rhpkg.errata] 22 | url = https://errata.devel.redhat.com/ 23 | 24 | [rhpkg.UMB] 25 | brokers = amqps://some-server.redhat.com:5671 amqps://some-server.ext.phx2.redhat.com:5671 26 | 27 | [rhpkg.test-rpmdiff] 28 | hub = https://rpmdiff-hub.host.prod.eng.bos.redhat.com/api/v1/ 29 | 30 | [rhpkg.bugzilla] 31 | url = https://bugzilla.redhat.com/xmlrpc.cgi 32 | 33 | [rhpkg.remotes] 34 | fedora_cli = fedpkg 35 | fedora_dist_git = pkgs.fedoraproject.org 36 | centos_cli = centpkg 37 | centos_dist_git = git.centos.org 38 | -------------------------------------------------------------------------------- /dist-git/beaker-tests/pkgs-files/lookaside-upload.conf: -------------------------------------------------------------------------------- 1 | 2 | # This alias must come before the /repo/ one to avoid being overridden. 3 | ScriptAlias /repo/pkgs/upload.cgi /var/lib/dist-git/web/upload.cgi 4 | 5 | Alias /repo/ /var/lib/dist-git/cache/lookaside/ 6 | 7 | ServerName pkgs.example.org 8 | ServerAdmin webmaster@fedoraproject.org 9 | 10 | LogLevel trace6 11 | 12 | # provide this manually for upload.cgi 13 | SetEnv SSL_CLIENT_S_DN_CN clime 14 | 15 | 16 | Options +ExecCGI 17 | Require all granted 18 | 19 | 20 | 21 | -------------------------------------------------------------------------------- /dist-git/beaker-tests/pkgs-files/ssl.conf: -------------------------------------------------------------------------------- 1 | # extracted from /etc/httpd/conf.d/ssl.conf for httpd-2.4.29 2 | # https://github.com/release-engineering/dist-git/issues/17 3 | 4 | Listen 443 https 5 | SSLPassPhraseDialog exec:/usr/libexec/httpd-ssl-pass-dialog 6 | SSLSessionCache shmcb:/run/httpd/sslcache(512000) 7 | SSLSessionCacheTimeout 300 8 | SSLRandomSeed startup file:/dev/urandom 256 9 | SSLRandomSeed connect builtin 10 | SSLCryptoDevice builtin 11 | -------------------------------------------------------------------------------- /dist-git/beaker-tests/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # vim: dict+=/usr/share/beakerlib/dictionary.vim cpt=.,w,b,u,t,i,k 3 | # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 4 | # 5 | # runtest.sh of /tools/dist-git/Regression/ 6 | # Description: Test dist-git functionality. 7 | # Author: clime 8 | # 9 | # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 10 | # 11 | # Copyright (c) 2016 Red Hat, Inc. 12 | # 13 | # This program is free software: you can redistribute it and/or 14 | # modify it under the terms of the GNU General Public License as 15 | # published by the Free Software Foundation, either version 2 of 16 | # the License, or (at your option) any later version. 17 | # 18 | # This program is distributed in the hope that it will be 19 | # useful, but WITHOUT ANY WARRANTY; without even the implied 20 | # warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR 21 | # PURPOSE. See the GNU General Public License for more details. 22 | # 23 | # You should have received a copy of the GNU General Public License 24 | # along with this program. If not, see http://www.gnu.org/licenses/. 25 | # 26 | # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 27 | 28 | while [[ $# > 0 ]] 29 | do 30 | key="$1" 31 | case $key in 32 | --nosetup|-x) 33 | nosetup=YES 34 | ;; 35 | --runonly|-r) 36 | shift 37 | runonly=$1 38 | ;; 39 | *) 40 | echo "Unknown option $key." 41 | exit 1 42 | ;; 43 | esac 44 | shift 45 | done 46 | 47 | # unnecessary on actual beaker machines but good for local docker testing 48 | if ! rpm -qa | grep -E '^rhts.*' &> /dev/null || ! rpm -qa | grep -E '.*beaker.*' &> /dev/null; then 49 | releasever=`cat /etc/redhat-release | awk '{print $3}'` 50 | sudo dnf -y --nogpgcheck --repofrompath=beakerrepo,http://beaker-project.org/yum/client/Fedora$releasever/ \ 51 | --enablerepo=beakerrepo install rhts-test-env beakerlib 52 | fi 53 | 54 | # include Beaker environment 55 | . /usr/bin/rhts-environment.sh || exit 1 56 | . /usr/share/beakerlib/beakerlib.sh || exit 1 57 | 58 | PACKAGE="dist-git" 59 | 60 | export TESTPATH="$( builtin cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 61 | 62 | shopt -s extglob 63 | 64 | rlJournalStart 65 | rlPhaseStartSetup 66 | if [[ ! $nosetup ]]; then 67 | ./setup.sh 68 | fi 69 | rlPhaseEnd 70 | 71 | rlPhaseStartTest Tests 72 | for t in $TESTPATH/tests/!(*.disable); do 73 | if [[ ! $runonly ]] || echo $runonly | grep `basename $t` &> /dev/null; then 74 | $t/run.sh 75 | fi 76 | done 77 | rlPhaseEnd 78 | 79 | rlPhaseStartCleanup 80 | rlPhaseEnd 81 | rlJournalPrintText 82 | rlJournalEnd 83 | -------------------------------------------------------------------------------- /dist-git/beaker-tests/setup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | export SCRIPTPATH="$( builtin cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 4 | export LANG=en_US.utf8 5 | 6 | # primarily install git for the setup below 7 | dnf -y install git 8 | 9 | if [[ `pwd` =~ ^/mnt/tests.*$ ]]; then 10 | echo "Setting up native beaker environment." 11 | git clone https://pagure.io/dist-git.git 12 | export DISTGITROOTDIR=$SCRIPTPATH/dist-git 13 | else 14 | echo "Setting up from source tree." 15 | export DISTGITROOTDIR=$SCRIPTPATH/../ 16 | fi 17 | 18 | # install files from 'files' 19 | cp -rT $SCRIPTPATH/files / 20 | 21 | # install stuff needed for the test 22 | dnf -y install vagrant vagrant-libvirt rsync jq git wget fedpkg rpkg 23 | 24 | # enable libvirtd for Vagrant (distgit) 25 | systemctl enable libvirtd && systemctl start libvirtd 26 | systemctl start virtlogd.socket # this is currently needed in f25 for vagrant to work with libvirtd 27 | 28 | cd $DISTGITROOTDIR 29 | 30 | if [ -z $DISTGIT_FLAVOR ]; then 31 | DISTGIT_FLAVOR=dist-git 32 | fi 33 | 34 | DISTGITSTATUS=`mktemp` 35 | vagrant status $DISTGIT_FLAVOR | tee $DISTGITSTATUS 36 | if grep -q 'not created' $DISTGITSTATUS; then 37 | vagrant up $DISTGIT_FLAVOR 38 | else 39 | vagrant reload $DISTGIT_FLAVOR 40 | fi 41 | rm $DISTGITSTATUS 42 | 43 | IPADDR=`vagrant ssh -c "ifconfig eth0 | grep -E 'inet\s' | sed 's/\s*inet\s*\([0-9.]*\).*/\1/'" $DISTGIT_FLAVOR` 44 | echo "$IPADDR pkgs.example.org" >> /etc/hosts 45 | 46 | if ! [ -f ~/.ssh/id_rsa ]; then 47 | mkdir -p ~/.ssh && chmod 700 ~/.ssh 48 | ssh-keygen -f ~/.ssh/id_rsa -N '' -q 49 | fi 50 | 51 | PUBKEY=`cat ~/.ssh/id_rsa.pub` 52 | vagrant ssh -c "echo $PUBKEY > /tmp/id_rsa.pub.remote" $DISTGIT_FLAVOR 53 | 54 | vagrant ssh -c ' 55 | sudo mkdir -p /home/clime/.ssh 56 | sudo cp /tmp/id_rsa.pub.remote /home/clime/.ssh/authorized_keys 57 | sudo chown -R clime:clime /home/clime/.ssh 58 | sudo chmod 700 /home/clime/.ssh 59 | sudo chmod 600 /home/clime/.ssh/authorized_keys 60 | ' $DISTGIT_FLAVOR 61 | 62 | vagrant ssh -c ' 63 | sudo mkdir -p /root/.ssh 64 | sudo cp /tmp/id_rsa.pub.remote /root/.ssh/authorized_keys 65 | sudo chmod 700 /root/.ssh 66 | sudo chmod 600 /root/.ssh/authorized_keys 67 | ' $DISTGIT_FLAVOR 68 | 69 | cd $SCRIPTPATH 70 | -------------------------------------------------------------------------------- /dist-git/beaker-tests/tests/basic-test/dist-git.conf: -------------------------------------------------------------------------------- 1 | [dist-git] 2 | git_author_name = Fedora Release Engineering 3 | git_author_email = rel-eng@lists.fedoraproject.org 4 | 5 | cache_dir = /var/lib/dist-git/cache 6 | gitroot_dir = /var/lib/dist-git/git 7 | 8 | gitolite = True 9 | grok = True 10 | 11 | default_namespace = rpms 12 | 13 | [upload] 14 | fedmsgs = True 15 | old_paths = True 16 | nomd5 = True 17 | -------------------------------------------------------------------------------- /dist-git/beaker-tests/tests/basic-test/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | . /usr/bin/rhts-environment.sh || exit 1 4 | . /usr/share/beakerlib/beakerlib.sh || exit 1 5 | 6 | export SCRIPTDIR="$( builtin cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 7 | export CWD=`pwd` 8 | export TEST_CWD=`mktemp -d` 9 | 10 | function pkgs_cmd { 11 | ssh -o 'StrictHostKeyChecking no' clime@pkgs.example.org $1 12 | } 13 | 14 | rlJournalStart 15 | rlPhaseStartSetup BasicTest 16 | scp -o 'StrictHostKeyChecking no' $SCRIPTDIR/dist-git.conf root@pkgs.example.org:/etc/dist-git/dist-git.conf 17 | pkgs_cmd 'git config --global user.email "clime@redhat.com"' 18 | pkgs_cmd 'git config --global user.name "clime"' 19 | pkgs_cmd '/usr/share/dist-git/setup_git_package rpms/prunerepo' 20 | rlPhaseEnd 21 | 22 | rlPhaseStartTest BasicTest 23 | cd $TEST_CWD 24 | echo "Running tests in $TEST_CWD" 25 | 26 | # clone repo using rpkg 27 | rlRun "rpkg -v clone rpms/prunerepo" 28 | 29 | cd prunerepo 30 | git config user.email "somebody@example.com" 31 | git config user.name "Some name" 32 | 33 | # upload into lookaside and working tree update 34 | rlRun "rpkg -v import --skip-diffs $SCRIPTDIR/../../data/prunerepo-1.1-1.fc23.src.rpm" 35 | 36 | # test of presence of the uploaded file 37 | rlRun 'wget http://pkgs.example.org/repo/pkgs/rpms/prunerepo/prunerepo-1.1.tar.gz/sha512/6a6a30c0e8c661176ba0cf7e8f1909a493a298fd5088389f5eb630b577dee157106e5f89dc429bcf2a6fdffe4bc10b498906b9746220882827560bc5f72a1b01/prunerepo-1.1.tar.gz' 38 | 39 | # commit of spec and updated sources and push into the git repo 40 | rlRun "git add -A && git commit -m 'test commit'" 41 | rlRun "rpkg push" 42 | 43 | # https://pagure.io/rpkg-client/issue/4 44 | rlRun "rpkg clean -x" 45 | 46 | # get srpm file using rpkg 47 | rlRun "rpkg srpm" 48 | 49 | cd .. 50 | 51 | # test git-daemon and git-http-backend by read access 52 | rlRun "git clone git://pkgs.example.org/rpms/prunerepo.git prunerepo-copy" 53 | rlRun "git clone http://pkgs.example.org/git/rpms/prunerepo prunerepo-copy2" 54 | 55 | # GROK-MIRROR related tests 56 | # The following tests are run only if grok-manifest command is present in the virtual machine 57 | if `pkgs_cmd 'command -v grok-manifest > /dev/null'`; then 58 | # test manifest file update 59 | rlRun "wget http://pkgs.example.org/manifest.js.gz" 60 | gunzip ./manifest.js.gz 61 | rlRun "cat manifest.js | grep prunerepo.git" 62 | mv ./manifest.js manifest.js.prev 63 | 64 | # clone repo using rpkg 65 | rlRun "rpkg clone rpms/prunerepo prunerepo2" 66 | 67 | cd prunerepo2 68 | echo "manifest test" > sources 69 | 70 | rlRun "git add -A && git commit -m 'test commit 2'" 71 | rlRun "git push" 72 | 73 | cd .. 74 | 75 | rlRun "wget http://pkgs.example.org/manifest.js.gz" 76 | gunzip ./manifest.js.gz 77 | rlRun "cat manifest.js | grep prunerepo.git" 78 | 79 | modified_prev=`jq '.["/rpms/prunerepo.git"].modified' manifest.js.prev` 80 | modified=`jq '.["/rpms/prunerepo.git"].modified' manifest.js` 81 | 82 | rlAssertGreater "Check that 'modified' timestamp has been updated in the manifest file" $modified $modified_prev 83 | else 84 | echo -e "\e[93mgrok-manifest not present on the target machine. Skipping grok-mirror related tests!\e[39m" 85 | fi 86 | 87 | cd $CWD 88 | rlPhaseEnd 89 | 90 | rlPhaseStartCleanup BasicTest 91 | pkgs_cmd 'rm -rf /var/lib/dist-git/git/rpms/prunerepo.git' 92 | pkgs_cmd 'sudo rm -rf /var/lib/dist-git/cache/lookaside/pkgs/rpms/prunerepo' 93 | rm -rf $TEST_CWD 94 | rlPhaseEnd 95 | rlJournalEnd &> /dev/null 96 | -------------------------------------------------------------------------------- /dist-git/beaker-tests/tests/direct-test/dist-git-no-namespace.conf: -------------------------------------------------------------------------------- 1 | [dist-git] 2 | git_author_name = Fedora Release Engineering 3 | git_author_email = rel-eng@lists.fedoraproject.org 4 | 5 | cache_dir = /var/lib/dist-git/cache 6 | gitroot_dir = /var/lib/dist-git/git 7 | 8 | gitolite = True 9 | grok = True 10 | 11 | [upload] 12 | fedmsgs = True 13 | old_paths = True 14 | nomd5 = True 15 | -------------------------------------------------------------------------------- /dist-git/beaker-tests/tests/direct-test/dist-git.conf: -------------------------------------------------------------------------------- 1 | [dist-git] 2 | git_author_name = Fedora Release Engineering 3 | git_author_email = rel-eng@lists.fedoraproject.org 4 | 5 | cache_dir = /var/lib/dist-git/cache 6 | gitroot_dir = /var/lib/dist-git/git 7 | 8 | gitolite = True 9 | grok = True 10 | 11 | default_namespace = rpms 12 | 13 | [upload] 14 | fedmsgs = True 15 | old_paths = True 16 | nomd5 = True 17 | -------------------------------------------------------------------------------- /dist-git/beaker-tests/tests/direct-test/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | . /usr/bin/rhts-environment.sh || exit 1 4 | . /usr/share/beakerlib/beakerlib.sh || exit 1 5 | 6 | export SCRIPTDIR="$( builtin cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 7 | export CWD=`pwd` 8 | export TEST_CWD=`mktemp -d` 9 | 10 | function pkgs_cmd { 11 | ssh -o 'StrictHostKeyChecking no' clime@pkgs.example.org $1 12 | } 13 | 14 | rlJournalStart 15 | rlPhaseStartSetup DirectTest 16 | scp -o 'StrictHostKeyChecking no' $SCRIPTDIR/dist-git.conf root@pkgs.example.org:/etc/dist-git/dist-git.conf 17 | rlPhaseEnd 18 | 19 | rlPhaseStartTest DirectTest 20 | cd $TEST_CWD 21 | echo "Running tests in $TEST_CWD" 22 | 23 | pkgs_cmd '/usr/share/dist-git/setup_git_package prunerepo' 24 | rlRun "git clone git://pkgs.example.org/rpms/prunerepo.git" 25 | 26 | resp=`curl --silent -X POST http://pkgs.example.org/repo/pkgs/upload.cgi` 27 | rlRun "grep -q 'Required field \"name\" is not present.' <<< '$resp'" 28 | 29 | resp=`curl --silent -X POST http://pkgs.example.org/repo/pkgs/upload.cgi -F name=prunerepo -F sha512sum=a5f31ae7586dae8dc1ca9a91df208893a0c3ab0032ab153c12eb4255f7219e4baec4c7581f353295c52522fee155c64f1649319044fd1bbb40451f123496b6b3 -F file=@"$SCRIPTDIR/../../data/prunerepo-1.1-1.fc23.src.rpm"` 30 | rlRun "grep -q 'stored OK' <<< '$resp'" 31 | 32 | # test of presence of the uploaded file 33 | rlRun 'wget http://pkgs.example.org/repo/pkgs/rpms/prunerepo/prunerepo-1.1-1.fc23.src.rpm/sha512/a5f31ae7586dae8dc1ca9a91df208893a0c3ab0032ab153c12eb4255f7219e4baec4c7581f353295c52522fee155c64f1649319044fd1bbb40451f123496b6b3/prunerepo-1.1-1.fc23.src.rpm' 34 | 35 | resp=`curl --silent -X POST http://pkgs.example.org/repo/pkgs/upload.cgi -F name=prunerepo -F sha512sum=a5f31ae7586dae8dc1ca9a91df208893a0c3ab0032ab153c12eb4255f7219e4baec4c7581f353295c52522fee155c64f1649319044fd1bbb40451f123496b6b3 -F filename=prunerepo-1.1-1.fc23.src.rpm` 36 | rlRun "grep -q 'Available' <<< '$resp'" 37 | 38 | # test an error is raised on a non-existing module 39 | resp=`curl --silent -X POST http://pkgs.example.org/repo/pkgs/upload.cgi -F name=foo -F md5sum=80e541f050d558424d62743195481595 -F file=@"$SCRIPTDIR/../../data/prunerepo-1.1-1.fc23.src.rpm"` 40 | rlRun "grep -q 'Module \"rpms/foo\" does not exist!' <<< '$resp'" 41 | 42 | # test that md5 is forbidden by default 43 | resp=`curl --silent -X POST http://pkgs.example.org/repo/pkgs/upload.cgi -F name=prunerepo -F md5sum=80e541f050d558424d62743195481595 -F file=@"$SCRIPTDIR/../../data/prunerepo-1.1-1.fc23.src.rpm"` 44 | rlRun "grep -q 'Uploads with md5 are no longer allowed.' <<< '$resp'" 45 | 46 | ### try operating without default namespacing 47 | scp -o 'StrictHostKeyChecking no' $SCRIPTDIR/dist-git-no-namespace.conf root@pkgs.example.org:/etc/dist-git/dist-git.conf 48 | 49 | pkgs_cmd '/usr/share/dist-git/setup_git_package prunerepo' 50 | rlRun "git clone git://pkgs.example.org/prunerepo.git prunerepo2" 51 | 52 | resp=`curl --silent -X POST http://pkgs.example.org/repo/pkgs/upload.cgi -F name=prunerepo -F sha512sum=a5f31ae7586dae8dc1ca9a91df208893a0c3ab0032ab153c12eb4255f7219e4baec4c7581f353295c52522fee155c64f1649319044fd1bbb40451f123496b6b3 -F file=@"$SCRIPTDIR/../../data/prunerepo-1.1-1.fc23.src.rpm"` 53 | rlRun "grep -q 'stored OK' <<< '$resp'" 54 | 55 | # test of presence of the uploaded file 56 | rlRun 'wget http://pkgs.example.org/repo/pkgs/prunerepo/prunerepo-1.1-1.fc23.src.rpm/sha512/a5f31ae7586dae8dc1ca9a91df208893a0c3ab0032ab153c12eb4255f7219e4baec4c7581f353295c52522fee155c64f1649319044fd1bbb40451f123496b6b3/prunerepo-1.1-1.fc23.src.rpm' 57 | 58 | # test an error is raised on a non-existing module 59 | resp=`curl --silent -X POST http://pkgs.example.org/repo/pkgs/upload.cgi -F name=foo -F md5sum=80e541f050d558424d62743195481595 -F file=@"$SCRIPTDIR/../../data/prunerepo-1.1-1.fc23.src.rpm"` 60 | rlRun "grep -q 'Module \"foo\" does not exist!' <<< '$resp'" 61 | 62 | # test that mtime timestamp is preserved if sent 63 | pkgs_cmd 'sudo rm -rf /var/lib/dist-git/cache/lookaside/pkgs/prunerepo' 64 | 65 | resp=`curl --silent -X POST http://pkgs.example.org/repo/pkgs/upload.cgi -F name=prunerepo -F sha512sum=a5f31ae7586dae8dc1ca9a91df208893a0c3ab0032ab153c12eb4255f7219e4baec4c7581f353295c52522fee155c64f1649319044fd1bbb40451f123496b6b3 -F file=@"$SCRIPTDIR/../../data/prunerepo-1.1-1.fc23.src.rpm" -F mtime=1234` 66 | rlRun "grep -q 'stored OK' <<< '$resp'" 67 | 68 | mtime_verbose=`curl -s --head http://pkgs.example.org/repo/pkgs/prunerepo/prunerepo-1.1-1.fc23.src.rpm/sha512/a5f31ae7586dae8dc1ca9a91df208893a0c3ab0032ab153c12eb4255f7219e4baec4c7581f353295c52522fee155c64f1649319044fd1bbb40451f123496b6b3/prunerepo-1.1-1.fc23.src.rpm | grep 'Last-Modified:' | sed -E 's/^Last-Modified:\s*(.*)/\1/'` 69 | mtime=`date +%s --date="$mtime_verbose"` 70 | rlAssertEquals "Verify that we got the correct timestamp back" $mtime 1234 71 | 72 | pkgs_cmd 'sudo rm -rf /var/lib/dist-git/cache/lookaside/pkgs/prunerepo' 73 | 74 | # Invalid mtime value returns 400 Bad Request 75 | code=`curl -w '%{response_code}' --output /dev/null --silent -X POST http://pkgs.example.org/repo/pkgs/upload.cgi -F name=prunerepo -F sha512sum=a5f31ae7586dae8dc1ca9a91df208893a0c3ab0032ab153c12eb4255f7219e4baec4c7581f353295c52522fee155c64f1649319044fd1bbb40451f123496b6b3 -F file=@"$SCRIPTDIR/../../data/prunerepo-1.1-1.fc23.src.rpm" -F mtime=invalid` 76 | rlAssertEquals "Verify that we got 400 error on invalid mtime value" $code 400 77 | 78 | cd $CWD 79 | rlPhaseEnd 80 | 81 | rlPhaseStartCleanup DirectTest 82 | pkgs_cmd 'rm -rf /var/lib/dist-git/git/rpms/prunerepo.git' 83 | pkgs_cmd 'rm -rf /var/lib/dist-git/git/prunerepo.git' 84 | pkgs_cmd 'sudo rm -rf /var/lib/dist-git/cache/lookaside/pkgs/rpms/prunerepo' 85 | pkgs_cmd 'sudo rm -rf /var/lib/dist-git/cache/lookaside/pkgs/prunerepo' 86 | rm -rf $TEST_CWD 87 | rlPhaseEnd 88 | rlJournalEnd &> /dev/null 89 | -------------------------------------------------------------------------------- /dist-git/beaker-tests/tests/fedpkg-lookaside_dir-test/dist-git.conf: -------------------------------------------------------------------------------- 1 | [dist-git] 2 | git_author_name = Fedora Release Engineering 3 | git_author_email = rel-eng@lists.fedoraproject.org 4 | 5 | cache_dir = /var/lib/dist-git/cache 6 | gitroot_dir = /var/lib/dist-git/git 7 | 8 | lookaside_dir = /var/lib/dist-git/cache/lookaside/pkgs/ns 9 | 10 | gitolite = True 11 | grok = True 12 | 13 | default_namespace = rpms 14 | 15 | [upload] 16 | fedmsgs = True 17 | old_paths = True 18 | nomd5 = True 19 | -------------------------------------------------------------------------------- /dist-git/beaker-tests/tests/fedpkg-lookaside_dir-test/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | . /usr/bin/rhts-environment.sh || exit 1 4 | . /usr/share/beakerlib/beakerlib.sh || exit 1 5 | 6 | export SCRIPTDIR="$( builtin cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 7 | export CWD=`pwd` 8 | export TEST_CWD=`mktemp -d` 9 | 10 | function pkgs_cmd { 11 | ssh -o 'StrictHostKeyChecking no' clime@pkgs.example.org $1 12 | } 13 | 14 | rlJournalStart 15 | rlPhaseStartSetup FedpkgTest 16 | scp -o 'StrictHostKeyChecking no' $SCRIPTDIR/dist-git.conf root@pkgs.example.org:/etc/dist-git/dist-git.conf 17 | pkgs_cmd 'git config --global user.email "clime@redhat.com"' 18 | pkgs_cmd 'git config --global user.name "clime"' 19 | pkgs_cmd '/usr/share/dist-git/setup_git_package rpms/prunerepo' 20 | rlPhaseEnd 21 | 22 | rlPhaseStartTest FedpkgTest 23 | cd $TEST_CWD 24 | echo "Running tests in $TEST_CWD" 25 | 26 | # clone repo using fedpkg 27 | rlRun "fedpkg -v clone rpms/prunerepo" 28 | 29 | cd prunerepo 30 | 31 | # upload into lookaside and working tree update 32 | rlRun "fedpkg -v import --skip-diffs $SCRIPTDIR/../../data/prunerepo-1.1-1.fc23.src.rpm" 33 | 34 | # test of presence of the uploaded file 35 | rlRun 'wget http://pkgs.example.org/repo/pkgs/ns/rpms/prunerepo/prunerepo-1.1.tar.gz/sha512/6a6a30c0e8c661176ba0cf7e8f1909a493a298fd5088389f5eb630b577dee157106e5f89dc429bcf2a6fdffe4bc10b498906b9746220882827560bc5f72a1b01/prunerepo-1.1.tar.gz' 36 | 37 | cd $CWD 38 | rlPhaseEnd 39 | 40 | rlPhaseStartCleanup FedpkgTest 41 | pkgs_cmd 'rm -rf /var/lib/dist-git/git/rpms/prunerepo.git' 42 | pkgs_cmd 'sudo rm -rf /var/lib/dist-git/cache/lookaside/pkgs/ns/rpms/prunerepo' 43 | rm -rf $TEST_CWD 44 | rlPhaseEnd 45 | rlJournalEnd &> /dev/null 46 | -------------------------------------------------------------------------------- /dist-git/beaker-tests/tests/fedpkg-test/dist-git.conf: -------------------------------------------------------------------------------- 1 | [dist-git] 2 | git_author_name = Fedora Release Engineering 3 | git_author_email = rel-eng@lists.fedoraproject.org 4 | 5 | cache_dir = /var/lib/dist-git/cache 6 | gitroot_dir = /var/lib/dist-git/git 7 | 8 | gitolite = True 9 | grok = True 10 | 11 | default_namespace = rpms 12 | 13 | [upload] 14 | fedmsgs = True 15 | old_paths = True 16 | nomd5 = True 17 | -------------------------------------------------------------------------------- /dist-git/beaker-tests/tests/fedpkg-test/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | . /usr/bin/rhts-environment.sh || exit 1 4 | . /usr/share/beakerlib/beakerlib.sh || exit 1 5 | 6 | export SCRIPTDIR="$( builtin cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 7 | export CWD=`pwd` 8 | export TEST_CWD=`mktemp -d` 9 | 10 | function pkgs_cmd { 11 | ssh -o 'StrictHostKeyChecking no' clime@pkgs.example.org $1 12 | } 13 | 14 | rlJournalStart 15 | rlPhaseStartSetup FedpkgTest 16 | scp -o 'StrictHostKeyChecking no' $SCRIPTDIR/dist-git.conf root@pkgs.example.org:/etc/dist-git/dist-git.conf 17 | pkgs_cmd 'git config --global user.email "clime@redhat.com"' 18 | pkgs_cmd 'git config --global user.name "clime"' 19 | pkgs_cmd '/usr/share/dist-git/setup_git_package rpms/prunerepo' 20 | rlPhaseEnd 21 | 22 | rlPhaseStartTest FedpkgTest 23 | cd $TEST_CWD 24 | echo "Running tests in $TEST_CWD" 25 | 26 | # clone repo using fedpkg 27 | rlRun "fedpkg -v clone rpms/prunerepo" 28 | 29 | cd prunerepo 30 | git config user.email "somebody@example.com" 31 | git config user.name "Some name" 32 | 33 | # upload into lookaside and working tree update 34 | rlRun "fedpkg -v import --skip-diffs $SCRIPTDIR/../../data/prunerepo-1.1-1.fc23.src.rpm" 35 | 36 | # test of presence of the uploaded file 37 | rlRun 'wget http://pkgs.example.org/repo/pkgs/rpms/prunerepo/prunerepo-1.1.tar.gz/sha512/6a6a30c0e8c661176ba0cf7e8f1909a493a298fd5088389f5eb630b577dee157106e5f89dc429bcf2a6fdffe4bc10b498906b9746220882827560bc5f72a1b01/prunerepo-1.1.tar.gz' 38 | 39 | # commit of spec and updated sources and push into the git repo 40 | rlRun "git add -A && git commit -m 'test commit'" 41 | rlRun "fedpkg push" 42 | 43 | # get srpm file using fedpkg 44 | rlRun "fedpkg --dist f27 srpm" 45 | 46 | cd $CWD 47 | rlPhaseEnd 48 | 49 | rlPhaseStartCleanup FedpkgTest 50 | pkgs_cmd 'rm -rf /var/lib/dist-git/git/rpms/prunerepo.git' 51 | pkgs_cmd 'sudo rm -rf /var/lib/dist-git/cache/lookaside/pkgs/rpms/prunerepo' 52 | rm -rf $TEST_CWD 53 | rlPhaseEnd 54 | rlJournalEnd &> /dev/null 55 | -------------------------------------------------------------------------------- /dist-git/beaker-tests/tests/group-check-test/dist-git-group-check-off.conf: -------------------------------------------------------------------------------- 1 | [dist-git] 2 | git_author_name = Fedora Release Engineering 3 | git_author_email = rel-eng@lists.fedoraproject.org 4 | 5 | cache_dir = /var/lib/dist-git/cache 6 | gitroot_dir = /var/lib/dist-git/git 7 | 8 | gitolite = True 9 | grok = True 10 | 11 | default_namespace = rpms 12 | 13 | [upload] 14 | fedmsgs = True 15 | old_paths = True 16 | nomd5 = True 17 | disable_group_check = True 18 | -------------------------------------------------------------------------------- /dist-git/beaker-tests/tests/group-check-test/dist-git-group-check-on.conf: -------------------------------------------------------------------------------- 1 | [dist-git] 2 | git_author_name = Fedora Release Engineering 3 | git_author_email = rel-eng@lists.fedoraproject.org 4 | 5 | cache_dir = /var/lib/dist-git/cache 6 | gitroot_dir = /var/lib/dist-git/git 7 | 8 | gitolite = True 9 | grok = True 10 | 11 | default_namespace = rpms 12 | 13 | [upload] 14 | fedmsgs = True 15 | old_paths = True 16 | nomd5 = True 17 | disable_group_check = False 18 | -------------------------------------------------------------------------------- /dist-git/beaker-tests/tests/group-check-test/dist-git-group-check-unset.conf: -------------------------------------------------------------------------------- 1 | [dist-git] 2 | git_author_name = Fedora Release Engineering 3 | git_author_email = rel-eng@lists.fedoraproject.org 4 | 5 | cache_dir = /var/lib/dist-git/cache 6 | gitroot_dir = /var/lib/dist-git/git 7 | 8 | gitolite = True 9 | grok = True 10 | 11 | default_namespace = rpms 12 | 13 | [upload] 14 | fedmsgs = True 15 | old_paths = True 16 | nomd5 = True 17 | -------------------------------------------------------------------------------- /dist-git/beaker-tests/tests/group-check-test/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | . /usr/bin/rhts-environment.sh || exit 1 4 | . /usr/share/beakerlib/beakerlib.sh || exit 1 5 | 6 | export SCRIPTDIR="$( builtin cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 7 | export CWD=`pwd` 8 | export TEST_CWD=`mktemp -d` 9 | 10 | function pkgs_cmd { 11 | ssh -o 'StrictHostKeyChecking no' clime@pkgs.example.org $1 12 | } 13 | 14 | function pkgs_root_cmd { 15 | ssh -o 'StrictHostKeyChecking no' root@pkgs.example.org $1 16 | } 17 | 18 | rlJournalStart 19 | rlPhaseStartSetup GroupCheckTest 20 | pkgs_root_cmd "gpasswd -a clime packager" 21 | rlPhaseEnd 22 | 23 | rlPhaseStartTest GroupCheckTest 24 | cd $TEST_CWD 25 | echo "Running tests in $TEST_CWD" 26 | 27 | ### DISABLE_GROUP_CHECK = False 28 | pkgs_root_cmd "gpasswd -a clime packager" 29 | scp -o 'StrictHostKeyChecking no' $SCRIPTDIR/dist-git-group-check-on.conf root@pkgs.example.org:/etc/dist-git/dist-git.conf 30 | 31 | pkgs_root_cmd "gpasswd -d clime packager" 32 | resp=`curl --silent -X POST http://pkgs.example.org/repo/pkgs/upload.cgi` 33 | rlRun "grep -q 'You must connect with a valid certificate and be in the packager group to upload.' <<< '$resp'" 34 | 35 | pkgs_root_cmd "gpasswd -a clime packager" 36 | resp=`curl --silent -X POST http://pkgs.example.org/repo/pkgs/upload.cgi` 37 | rlRun "grep -q 'Required field \"name\" is not present.' <<< '$resp'" 38 | 39 | ### DISABLE_GROUP_CHECK = True 40 | pkgs_root_cmd "gpasswd -a clime packager" 41 | scp -o 'StrictHostKeyChecking no' $SCRIPTDIR/dist-git-group-check-off.conf root@pkgs.example.org:/etc/dist-git/dist-git.conf 42 | 43 | pkgs_root_cmd "gpasswd -d clime packager" 44 | resp=`curl --silent -X POST http://pkgs.example.org/repo/pkgs/upload.cgi` 45 | rlRun "grep -q 'Required field \"name\" is not present.' <<< '$resp'" 46 | 47 | pkgs_root_cmd "gpasswd -a clime packager" 48 | resp=`curl --silent -X POST http://pkgs.example.org/repo/pkgs/upload.cgi` 49 | rlRun "grep -q 'Required field \"name\" is not present.' <<< '$resp'" 50 | 51 | ### DISABLE_GROUP_CHECK unset 52 | pkgs_root_cmd "gpasswd -a clime packager" 53 | scp -o 'StrictHostKeyChecking no' $SCRIPTDIR/dist-git-group-check-unset.conf root@pkgs.example.org:/etc/dist-git/dist-git.conf 54 | 55 | pkgs_root_cmd "gpasswd -d clime packager" 56 | resp=`curl --silent -X POST http://pkgs.example.org/repo/pkgs/upload.cgi` 57 | rlRun "grep -q 'You must connect with a valid certificate and be in the packager group to upload.' <<< '$resp'" 58 | 59 | pkgs_root_cmd "gpasswd -a clime packager" 60 | resp=`curl --silent -X POST http://pkgs.example.org/repo/pkgs/upload.cgi` 61 | rlRun "grep -q 'Required field \"name\" is not present.' <<< '$resp'" 62 | 63 | cd $CWD 64 | rlPhaseEnd 65 | 66 | rlPhaseStartCleanup GroupCheckTest 67 | rm -rf $TEST_CWD 68 | rlPhaseEnd 69 | rlJournalEnd &> /dev/null 70 | -------------------------------------------------------------------------------- /dist-git/beaker-tests/tests/rhpkg-test.disable/dist-git.conf: -------------------------------------------------------------------------------- 1 | [dist-git] 2 | git_author_name = Release Engineering 3 | git_author_email = rel-eng@redhat.com 4 | 5 | cache_dir = /var/lib/dist-git/cache 6 | gitroot_dir = /var/lib/dist-git/git 7 | 8 | gitolite = False 9 | grok = False 10 | 11 | default_namespace = rpms 12 | 13 | [upload] 14 | fedmsgs = False 15 | old_paths = True 16 | nomd5 = False 17 | -------------------------------------------------------------------------------- /dist-git/beaker-tests/tests/rhpkg-test.disable/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | . /usr/bin/rhts-environment.sh || exit 1 4 | . /usr/share/beakerlib/beakerlib.sh || exit 1 5 | 6 | export SCRIPTDIR="$( builtin cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 7 | export CWD=`pwd` 8 | export TEST_CWD=`mktemp -d` 9 | 10 | function pkgs_cmd { 11 | ssh -o 'StrictHostKeyChecking no' clime@pkgs.example.org $1 12 | } 13 | 14 | rlJournalStart 15 | rlPhaseStartSetup rhpkgTest 16 | scp -o 'StrictHostKeyChecking no' $SCRIPTDIR/dist-git.conf root@pkgs.example.org:/etc/dist-git/dist-git.conf 17 | pkgs_cmd 'git config --global user.email "clime@redhat.com"' 18 | pkgs_cmd 'git config --global user.name "clime"' 19 | pkgs_cmd '/usr/share/dist-git/setup_git_package rpms/prunerepo' 20 | rlPhaseEnd 21 | 22 | rlPhaseStartTest rhpkgTest 23 | cd $TEST_CWD 24 | echo "Running tests in $TEST_CWD" 25 | 26 | # clone repo using rhpkg 27 | rlRun "rhpkg -v clone rpms/prunerepo" 28 | 29 | cd prunerepo 30 | git config user.email "somebody@example.com" 31 | git config user.name "Some name" 32 | 33 | # upload into lookaside and working tree update 34 | rlRun "rhpkg -v import --skip-diffs $SCRIPTDIR/../../data/prunerepo-1.1-1.fc23.src.rpm" 35 | 36 | # test of presence of the uploaded file 37 | rlRun 'wget http://pkgs.example.org/repo/pkgs/rpms/prunerepo/prunerepo-1.1.tar.gz/c5af09c7fb2c05e556898c93c62b1e35/prunerepo-1.1.tar.gz' 38 | 39 | # commit of spec and updated sources and push into the git repo 40 | rlRun "git add -A && git commit -m 'test commit'" 41 | rlRun "rhpkg push" 42 | 43 | # delete imported tarball 44 | rm prunerepo-1.1.tar.gz 45 | 46 | # get srpm file using rhpkg 47 | rlRun "rhpkg -v --release rhel-8 srpm" 48 | 49 | cd $CWD 50 | rlPhaseEnd 51 | 52 | rlPhaseStartCleanup rhpkgTest 53 | pkgs_cmd 'rm -rf /var/lib/dist-git/git/rpms/prunerepo.git' 54 | pkgs_cmd 'sudo rm -rf /var/lib/dist-git/cache/lookaside/pkgs/rpms/prunerepo' 55 | rm -rf $TEST_CWD 56 | rlPhaseEnd 57 | rlJournalEnd &> /dev/null 58 | -------------------------------------------------------------------------------- /dist-git/beaker-tests/tests/test-template/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | . /usr/bin/rhts-environment.sh || exit 1 4 | . /usr/share/beakerlib/beakerlib.sh || exit 1 5 | 6 | export TESTPATH="$( builtin cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 7 | 8 | rlJournalStart 9 | rlPhaseStartTest TestTemplate 10 | # write your test here 11 | rlPhaseEnd 12 | rlJournalEnd &> /dev/null 13 | -------------------------------------------------------------------------------- /dist-git/configs/dist-git/dist-git.conf: -------------------------------------------------------------------------------- 1 | [dist-git] 2 | git_author_name = Fedora Release Engineering 3 | git_author_email = rel-eng@lists.fedoraproject.org 4 | 5 | # deprecated: 6 | # cache_dir = /var/lib/dist-git/cache 7 | 8 | lookaside_dir = /var/lib/dist-git/cache/lookaside/pkgs 9 | gitroot_dir = /var/lib/dist-git/git 10 | # relates to the dist-git-gc script (and timer), 11 | # we need to dive 2 levels from gitroot_dir in the directory structure 12 | git_gc_depth = 2 13 | 14 | gitolite = True 15 | grok = True 16 | 17 | default_namespace = rpms 18 | 19 | # name of the default branch (a.k.a. master or main) 20 | default_branch = master 21 | 22 | [upload] 23 | fedmsgs = True 24 | old_paths = True 25 | nomd5 = True 26 | disable_group_check = False 27 | -------------------------------------------------------------------------------- /dist-git/configs/httpd/dist-git.conf: -------------------------------------------------------------------------------- 1 | include "conf.d/dist-git/*.conf" 2 | -------------------------------------------------------------------------------- /dist-git/configs/httpd/dist-git/git-smart-http.conf: -------------------------------------------------------------------------------- 1 | SetEnv GIT_PROJECT_ROOT /var/lib/dist-git/git 2 | SetEnv GIT_HTTP_EXPORT_ALL 3 | ScriptAlias /git/ /usr/libexec/git-core/git-http-backend/ 4 | 5 | 6 | Options ExecCGI Indexes 7 | Order allow,deny 8 | Allow from all 9 | Require all granted 10 | 11 | -------------------------------------------------------------------------------- /dist-git/configs/httpd/dist-git/lookaside-upload.conf.example: -------------------------------------------------------------------------------- 1 | # default SSL configuration... 2 | Listen 443 3 | 4 | SSLSessionCache shmcb:/var/cache/mod_ssl/scache(512000) 5 | SSLSessionCacheTimeout 300 6 | 7 | Mutex default 8 | 9 | SSLRandomSeed startup file:/dev/urandom 256 10 | SSLRandomSeed connect builtin 11 | SSLCryptoDevice builtin 12 | 13 | # SSL host 14 | 15 | # This alias must come before the /repo/ one to avoid being overridden. 16 | ScriptAlias /repo/pkgs/upload.cgi /var/lib/dist-git/web/upload.cgi 17 | 18 | Alias /repo/ /var/lib/dist-git/cache/lookaside/ 19 | ServerName pkgs.example.org 20 | ServerAdmin webmaster@fedoraproject.org 21 | 22 | SSLEngine on 23 | 24 | SSLCertificateFile /etc/pki/tls/certs/pkgs.example.org.pem 25 | SSLCertificateKeyFile /etc/pki/tls/certs/pkgs.example.org.pem 26 | SSLCACertificateFile /etc/pki/tls/certs/ca-bundle.crt 27 | #SSLCARevocationFile /etc/pki/tls/certs/crl.pem 28 | 29 | SSLProtocol -All +TLSv1 +TLSv1.1 +TLSv1.2 30 | SSLCipherSuite ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!3DES:!MD5:!PSK 31 | 32 | # Must be 'optional' everywhere in order to have POST operations work to upload.cgi 33 | SSLVerifyClient optional 34 | # Must be here for POST operations to upload.cgi 35 | SSLOptions +OptRenegotiate 36 | ErrorLog logs/ssl_error_log 37 | CustomLog logs/ssl_access_log \ 38 | "%t %h %{SSL_PROTOCOL}x %{SSL_CIPHER}x \"%{SSL_CLIENT_S_DN_OU}x\" %{SSL_CLIENT_S_DN_CN}x %{SSL_CLIENT_S_DN_emailAddress}x \"%r\" %b" 39 | 40 | 41 | SSLVerifyClient optional 42 | SSLVerifyDepth 1 43 | SSLOptions +StrictRequire +StdEnvVars +OptRenegotiate 44 | # require that the client auth cert was created by us and signed by us 45 | SSLRequire ( %{SSL_CIPHER} !~ m/^(EXP|NULL)/ \ 46 | and %{SSL_CLIENT_S_DN_O} eq "Example Project" \ 47 | and %{SSL_CLIENT_S_DN_OU} eq "Example User Cert" \ 48 | and %{SSL_CLIENT_I_DN_O} eq "Example Project" \ 49 | and %{SSL_CLIENT_I_DN_OU} eq "Example Project CA" ) 50 | 51 | 52 | 53 | SSLRequireSSL 54 | 55 | Options +ExecCGI 56 | Require all granted 57 | 58 | SSLVerifyClient optional 59 | SSLVerifyDepth 1 60 | SSLOptions +StrictRequire +StdEnvVars +OptRenegotiate 61 | # require that the access comes from internal or that 62 | # the client auth cert was created by us and signed by us 63 | SSLRequire ( %{SSL_CIPHER} !~ m/^(EXP|NULL)/ \ 64 | and %{SSL_CLIENT_S_DN_O} eq "Example Project" \ 65 | and %{SSL_CLIENT_S_DN_OU} eq "Example User Cert" \ 66 | and %{SSL_CLIENT_I_DN_O} eq "Example Project" \ 67 | and %{SSL_CLIENT_I_DN_OU} eq "Example Project CA" ) 68 | 69 | 70 | 71 | -------------------------------------------------------------------------------- /dist-git/configs/httpd/dist-git/lookaside.conf: -------------------------------------------------------------------------------- 1 | Alias /lookaside /var/lib/dist-git/cache/lookaside 2 | Alias /repo /var/lib/dist-git/cache/lookaside 3 | 4 | Options Indexes FollowSymLinks 5 | AllowOverride None 6 | Require all granted 7 | 8 | 9 | -------------------------------------------------------------------------------- /dist-git/configs/httpd/dist-git/manifest.conf: -------------------------------------------------------------------------------- 1 | Alias /manifest.js.gz /var/lib/dist-git/git/manifest.js.gz 2 | 3 | 4 | Require all granted 5 | 6 | -------------------------------------------------------------------------------- /dist-git/configs/systemd/dist-git-gc.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Git Repositories Clean Up 3 | 4 | [Service] 5 | Type=oneshot 6 | ExecStart=/usr/bin/dist-git-gc 7 | 8 | [Install] 9 | WantedBy=multi-user.target 10 | -------------------------------------------------------------------------------- /dist-git/configs/systemd/dist-git-gc.timer: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Git Repositories Clean Up Timer 3 | 4 | [Timer] 5 | OnCalendar=monthly 6 | 7 | [Install] 8 | WantedBy=timers.target 9 | -------------------------------------------------------------------------------- /dist-git/configs/systemd/dist-git.socket: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Dist Git Activation Socket 3 | 4 | [Socket] 5 | ListenStream=9418 6 | Accept=true 7 | 8 | [Install] 9 | WantedBy=sockets.target 10 | -------------------------------------------------------------------------------- /dist-git/configs/systemd/dist-git@.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Git Repositories Server Daemon 3 | Documentation=man:git-daemon(1) 4 | Wants=dist-git.socket 5 | 6 | [Service] 7 | User=nobody 8 | ExecStart=/usr/libexec/git-core/git-daemon --base-path=/var/lib/dist-git/git --export-all --user-path=public_git --syslog --inetd --verbose 9 | StandardInput=socket 10 | -------------------------------------------------------------------------------- /dist-git/configs/sysusers.d/dist-git.conf: -------------------------------------------------------------------------------- 1 | g packager - 2 | -------------------------------------------------------------------------------- /dist-git/dist-git.spec: -------------------------------------------------------------------------------- 1 | %global selinux_variants mls targeted 2 | %global modulename dist_git 3 | %global installdir /var/lib/dist-git 4 | 5 | Name: dist-git 6 | Version: 1.18 7 | Release: 1%{?dist} 8 | Summary: Package source version control system 9 | 10 | # upload.cgi uses GPLv1 11 | License: MIT AND GPL-1.0-only 12 | URL: https://github.com/release-engineering/dist-git 13 | # Source is created by 14 | # git clone https://github.com/release-engineering/dist-git.git 15 | # cd dist-git 16 | # tito build --tgz 17 | Source0: %{name}-%{version}.tar.gz 18 | BuildArch: noarch 19 | 20 | BuildRequires: systemd 21 | 22 | Requires: httpd 23 | Requires: perl(Sys::Syslog) 24 | Requires: (dist-git-selinux if selinux-policy-targeted) 25 | Requires: git 26 | Requires: git-daemon 27 | Requires: mod_ssl 28 | Requires: crudini 29 | %if 0%{?rhel} && 0%{?rhel} < 10 30 | Requires(pre): shadow-utils 31 | %endif 32 | 33 | Requires: python3-requests 34 | Recommends: python3-grokmirror 35 | Suggests: python3-fedmsg 36 | Suggests: fedora-messaging 37 | %if 0%{?rhel} == 8 38 | BuildRequires: python3-nose 39 | %else 40 | BuildRequires: python3-pytest 41 | %endif 42 | BuildRequires: python3-parameterized 43 | BuildRequires: python3-requests 44 | 45 | # this should be Requires but see https://bugzilla.redhat.com/show_bug.cgi?id=1833810 46 | Recommends: moreutils 47 | 48 | %if 0%{?fedora} && 0%{?fedora} >= 41 49 | # The `cgi` module was removed from the Python 3.13 standard library 50 | BuildRequires: python3-legacy-cgi 51 | Requires: python3-legacy-cgi 52 | %endif 53 | 54 | 55 | %description 56 | DistGit is a Git repository specifically designed to hold RPM 57 | package sources. 58 | 59 | 60 | %package selinux 61 | Summary: SELinux support for dist-git 62 | 63 | BuildRequires: checkpolicy 64 | BuildRequires: hardlink 65 | 66 | Requires: %name = %version-%release 67 | BuildRequires: selinux-policy-devel 68 | %{?selinux_requires} 69 | 70 | 71 | %description selinux 72 | Dist Git is a remote Git repository specifically designed to hold RPM 73 | package sources. 74 | 75 | This package includes SELinux support. 76 | 77 | 78 | %prep 79 | %setup -q 80 | 81 | 82 | %build 83 | # ------------------------------------------------------------------------------ 84 | # SELinux 85 | # ------------------------------------------------------------------------------ 86 | cd selinux 87 | for selinuxvariant in %{selinux_variants} 88 | do 89 | make NAME=${selinuxvariant} -f /usr/share/selinux/devel/Makefile 90 | mv %{modulename}.pp %{modulename}.pp.${selinuxvariant} 91 | make NAME=${selinuxvariant} -f /usr/share/selinux/devel/Makefile clean 92 | done 93 | cd - 94 | 95 | 96 | %pre 97 | %if 0%{?rhel} && 0%{?rhel} < 10 98 | # ------------------------------------------------------------------------------ 99 | # Users and Groups 100 | # ------------------------------------------------------------------------------ 101 | getent group packager > /dev/null || \ 102 | groupadd -r packager 103 | exit 0 104 | %endif 105 | 106 | %check 107 | %if 0%{?rhel} && 0%{?rhel} <= 8 108 | %if 0%{?rhel} < 8 109 | nosetests -v . 110 | %else 111 | nosetests-3 -v . 112 | %endif 113 | %else 114 | pytest -vv . 115 | %endif 116 | 117 | 118 | %install 119 | # ------------------------------------------------------------------------------ 120 | # /usr/share/ ........... scripts 121 | # ------------------------------------------------------------------------------ 122 | install -d %{buildroot}%{_datadir}/dist-git/ 123 | cp -a scripts/dist-git/* %{buildroot}%{_datadir}/dist-git/ 124 | 125 | # ------------------------------------------------------------------------------ 126 | # /etc/ .......... config files 127 | # ------------------------------------------------------------------------------ 128 | install -d %{buildroot}%{_sysconfdir}/dist-git 129 | cp -a configs/dist-git/dist-git.conf %{buildroot}%{_sysconfdir}/dist-git/ 130 | install -d %{buildroot}%{_sysconfdir}/httpd/conf.d/dist-git 131 | mkdir -p %{buildroot}%{_unitdir} 132 | 133 | cp -a configs/httpd/dist-git.conf %{buildroot}%{_sysconfdir}/httpd/conf.d/ 134 | cp -a configs/httpd/dist-git/* %{buildroot}%{_sysconfdir}/httpd/conf.d/dist-git/ 135 | cp -a configs/systemd/* %{buildroot}%{_unitdir}/ 136 | 137 | install -m0644 -D configs/sysusers.d/dist-git.conf %{buildroot}%{_sysusersdir}/dist-git.conf 138 | 139 | # ------------------------------------------------------------------------------ 140 | # /var/lib/ ...... dynamic persistent files 141 | # ------------------------------------------------------------------------------ 142 | install -d %{buildroot}%{installdir} 143 | install -d %{buildroot}%{installdir}/git 144 | install -d %{buildroot}%{installdir}/cache 145 | install -d %{buildroot}%{installdir}/cache/lookaside 146 | install -d %{buildroot}%{installdir}/cache/lookaside/pkgs 147 | install -d %{buildroot}%{installdir}/web 148 | 149 | cp -a scripts/httpd/upload.cgi %{buildroot}%{installdir}/web/ 150 | 151 | %if 0%{?rhel} && 0%{?rhel} < 8 152 | sed -i '1 s|#.*|#!/usr/bin/python2|' %{buildroot}%{installdir}/web/upload.cgi 153 | %endif 154 | 155 | # ------------------------------------------------------------------------------ 156 | # /usr/bin/ ...... links to executable files 157 | # ------------------------------------------------------------------------------ 158 | install -d %{buildroot}%{_bindir} 159 | ln -s %{_datadir}/dist-git/setup_git_package %{buildroot}%{_bindir}/setup_git_package 160 | ln -s %{_datadir}/dist-git/mkbranch %{buildroot}%{_bindir}/mkbranch 161 | ln -s %{_datadir}/dist-git/mkbranch_branching %{buildroot}%{_bindir}/mkbranch_branching 162 | ln -s %{_datadir}/dist-git/remove_unused_sources %{buildroot}%{_bindir}/remove_unused_sources 163 | mv %{buildroot}%{_datadir}/dist-git/dist-git-gc %{buildroot}%{_bindir}/dist-git-gc 164 | 165 | # ------------------------------------------------------------------------------ 166 | # SELinux 167 | # ------------------------------------------------------------------------------ 168 | cd selinux 169 | for selinuxvariant in %{selinux_variants} 170 | do 171 | install -d %{buildroot}%{_datadir}/selinux/${selinuxvariant} 172 | install -p -m 644 %{modulename}.pp.${selinuxvariant} \ 173 | %{buildroot}%{_datadir}/selinux/${selinuxvariant}/%{modulename}.pp 174 | done 175 | cd - 176 | 177 | hardlink -cv %{buildroot}%{_datadir}/selinux 178 | 179 | %post selinux 180 | for selinuxvariant in %{selinux_variants} 181 | do 182 | /usr/sbin/semodule -s ${selinuxvariant} -i \ 183 | %{_datadir}/selinux/${selinuxvariant}/%{modulename}.pp &> /dev/null || : 184 | done 185 | %{_sbindir}/restorecon -v %{installdir}/cache || : 186 | %{_sbindir}/restorecon -v %{installdir}/cache/lookaside || : 187 | %{_sbindir}/restorecon -v %{installdir}/cache/lookaside/pkgs || : 188 | %{_sbindir}/restorecon -v %{installdir}/git || : 189 | %{_sbindir}/restorecon -Rv %{installdir}/web/ || : 190 | 191 | %systemd_post dist-git.socket 192 | 193 | %preun 194 | %systemd_preun dist-git.socket 195 | 196 | %postun selinux 197 | if [ $1 -eq 0 ] ; then 198 | for selinuxvariant in %{selinux_variants} 199 | do 200 | /usr/sbin/semodule -s ${selinuxvariant} -r %{modulename} &> /dev/null || : 201 | done 202 | fi 203 | 204 | %systemd_postun dist-git.socket 205 | 206 | 207 | %files 208 | # ------------------------------------------------------------------------------ 209 | # Docs 210 | # ------------------------------------------------------------------------------ 211 | %license LICENSE 212 | %doc README.md 213 | 214 | # ------------------------------------------------------------------------------ 215 | # /etc/ .......... config files 216 | # ------------------------------------------------------------------------------ 217 | %dir %{_sysconfdir}/dist-git 218 | %config(noreplace) %{_sysconfdir}/dist-git/dist-git.conf 219 | %dir %{_sysconfdir}/httpd/conf.d/dist-git 220 | %config(noreplace) %{_sysconfdir}/httpd/conf.d/dist-git/* 221 | %config(noreplace) %{_sysconfdir}/httpd/conf.d/dist-git.conf 222 | 223 | %{_unitdir}/dist-git@.service 224 | %{_unitdir}/dist-git.socket 225 | %{_unitdir}/dist-git-gc.service 226 | %{_unitdir}/dist-git-gc.timer 227 | 228 | # ------------------------------------------------------------------------------ 229 | # /var/lib/ ...... dynamic persistent files 230 | # ------------------------------------------------------------------------------ 231 | 232 | # non-standard-dir-perm: 233 | # - git repositories and their contents must have w permission for their creators 234 | %dir %{installdir} 235 | %attr (2775, -, packager) %{installdir}/git 236 | %dir %{installdir}/web 237 | %attr (755, apache, apache) %{installdir}/web/upload.cgi 238 | %dir %{installdir}/cache 239 | %dir %{installdir}/cache/lookaside 240 | %attr (2775, apache, apache) %{installdir}/cache/lookaside/pkgs 241 | 242 | # ------------------------------------------------------------------------------ 243 | # /usr/share ...... executable files 244 | # ------------------------------------------------------------------------------ 245 | 246 | %dir %{_datadir}/dist-git 247 | %attr (775, -, -) %{_datadir}/dist-git/* 248 | %{_bindir}/dist-git-gc 249 | # TODO: move _datadir executables to _libexecdir, and drop these symlinks 250 | %{_bindir}/mkbranch 251 | %{_bindir}/mkbranch_branching 252 | %{_bindir}/remove_unused_sources 253 | %{_bindir}/setup_git_package 254 | 255 | %{_sysusersdir}/dist-git.conf 256 | 257 | %files selinux 258 | %doc selinux/* 259 | %{_datadir}/selinux/*/%{modulename}.pp 260 | 261 | 262 | %changelog 263 | * Fri Sep 27 2024 Pavel Raiskup 1.18-1 264 | - replace test on _selinux_policy_version by macro that does it all 265 | - handle situation when _selinux_policy_version is not defined 266 | - silence pylint 267 | - remove rhel7 specific code 268 | - Move %%_bindir symlinks out from dist-git-selinux 269 | - Depend on python3-legacy-cgi for F41+ 270 | - licensing: update the project's COPYING file 271 | - dist-git: move the code into sub-directory 272 | 273 | * Wed Jan 04 2023 Miroslav Suchý 1.17-1 274 | - use spdx license 275 | - use pytest instead of nose test 276 | 277 | * Mon Oct 5 2020 clime - 1.16-1 278 | - fixed exceptions for fedora-messaging 279 | - fixed topic for fedora-messaging (to git.lookaside.new) 280 | - spec tweak not to require distgit-selinux in containers 281 | - only Suggests fedmsg 282 | - garbage collection (git gc) script and systemd timer 283 | for its periodic run added 284 | 285 | * Fri Jun 12 2020 clime - 1.15-1 286 | - added support for fedora-messaging 287 | 288 | * Thu May 21 2020 clime - 1.14-1 289 | - disable unit tests temporarily as they do not pass 290 | with network disabled during build 291 | 292 | * Thu May 21 2020 clime - 1.13-1 293 | - fedmsg made an optional (recommnended) dependency 294 | - slight refactoring of upload.cgi script 295 | - tests added and documented 296 | 297 | * Tue May 28 2019 Miroslav Suchy - 1.12-1 298 | - remove old changelog entries 299 | - do not specify full path for hardlink [RHBZ#1714637] 300 | - add script for removing unused source tarballs in lookaside cache 301 | 302 | * Tue Apr 30 2019 clime 1.11-1 303 | - remove python3-configparser require 304 | - move scripts to bindir 305 | 306 | * Mon Mar 11 2019 clime 1.10-1 307 | - python3 support 308 | - fix post-receive hook in case post.receive.d is empty 309 | 310 | * Fri Nov 23 2018 clime 1.9-1 311 | - do not create sources file when creating a repo 312 | - set umask 0002 in all available dist-git scripts 313 | - Use REMOTE_USER as fallback for GSS_NAME 314 | - add support for setting mtime for an uploaded file 315 | 316 | * Tue Aug 14 2018 clime 1.8-1 317 | - add disable group check option 318 | - add lookaside_dir option 319 | - deprecate cache_dir 320 | - fix python-grokmirror dep 321 | 322 | * Mon Feb 26 2018 clime 1.7-1 323 | - move 'fedmsgs', 'old_paths', 'nomd5' options to optional upload section 324 | 325 | * Mon Feb 19 2018 clime 1.6-1 326 | - add 'fedmsgs', 'old_paths', and 'default_namespace' config options 327 | - remove domain_read_all_domains_state SELinux rule 328 | - require dist-git-selinux 329 | - give optional map permission to git_system_t on git_user_content_t 330 | - update requires to work for all environments 331 | - make the package completely distribution-agnostic 332 | 333 | * Mon Dec 18 2017 clime 1.5-1 334 | - make selinux policy build on f27+ 335 | - add optional map SELinux permission for httpd_t 336 | 337 | * Tue Jul 25 2017 clime 1.4-1 338 | - disable md5 uploading by default 339 | 340 | * Mon Jun 26 2017 clime 1.3-1 341 | - translate '/' to '-' in package name for mailinglist hook 342 | (graybrandon@gmail.com) 343 | 344 | * Fri May 26 2017 clime 1.2-1 345 | - remove mail git hook 346 | - grokmirror support 347 | 348 | * Wed May 03 2017 clime 1.1-1 349 | - fix default config value for email 350 | - fix name/email switch 351 | -------------------------------------------------------------------------------- /dist-git/docs/scripts/httpd/dist-git-stg.conf: -------------------------------------------------------------------------------- 1 | [dist-git] 2 | git_author_name = Release Engineering 3 | git_author_email = rel-eng@redhat.com 4 | 5 | cache_dir = /srv/cache 6 | lookaside_dir = /srv/cache/lookaside/pkgs 7 | 8 | gitroot_dir = /srv/git 9 | 10 | gitolite = False 11 | grok = False 12 | 13 | default_namespace = rpms 14 | 15 | [upload] 16 | fedmsgs = False 17 | old_paths = True 18 | nomd5 = False 19 | disable_group_check = True 20 | -------------------------------------------------------------------------------- /dist-git/docs/scripts/httpd/upload-rh.cgi: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # 3 | # CGI script to handle file updates for the rpms git repository. There 4 | # is nothing really complex here other than tedious checking of our 5 | # every step along the way... 6 | # 7 | 8 | import os 9 | import sys 10 | import errno 11 | import cgi 12 | import stat 13 | import hashlib 14 | import tempfile 15 | 16 | # reading buffer size 17 | BUFFER_SIZE = 4096 18 | GITREPO = "/srv/git" 19 | DEFAULT_NAMESPACE = 'rpms' 20 | 21 | # Lookaside cache directory 22 | CACHE_DIR = '/srv/cache/lookaside' 23 | 24 | form = cgi.FieldStorage() 25 | os.umask(002) 26 | 27 | def log_msg(*msgs): 28 | sys.stderr.write(' '.join(map(str,msgs)) + '\n') 29 | 30 | # abort running the script 31 | def send_error(text): 32 | print "Content-type: text/plain\n" 33 | print text 34 | sys.exit(1) 35 | 36 | # prepare to exit graciously 37 | def send_ok(text): 38 | print "Content-Type: text/plain" 39 | print 40 | if text: 41 | print text 42 | 43 | # check and validate that all the fields are present 44 | def check_form(var, strip=True): 45 | if not form.has_key(var): 46 | send_error("required field '%s' is not present" % (var,)) 47 | ret = form.getvalue(var) 48 | if type(ret) == type([]): 49 | send_error("Multiple values given for '%s'. Aborting" % (var,)) 50 | if strip: 51 | ret = os.path.basename(ret) # this is a path component 52 | return ret 53 | 54 | 55 | def hardlink(src, dst): 56 | """Create a hardlink, making sure the target directory exists.""" 57 | makedirs(os.path.dirname(dst)) 58 | if os.path.exists(dst): 59 | # The file already exists, let's hardlink over it. 60 | os.unlink(dst) 61 | os.link(src, dst) 62 | log_msg('ln %s %s' % (src, dst)) 63 | 64 | 65 | def makedirs(path, mode=0o2775): 66 | """Create a directory with all parents. If it already exists, then do 67 | nothing.""" 68 | try: 69 | os.makedirs(path, mode=mode) 70 | log_msg('mkdir -p %s' % path) 71 | except OSError as exc: 72 | if exc.errno != errno.EEXIST: 73 | send_error('Failed to create directory: %s' % exc) 74 | 75 | 76 | def check_file_exists(path, filename, upload): 77 | """Send message if exists and exit the script.""" 78 | if not os.access(path, os.F_OK | os.R_OK): 79 | # File does not exist, nothing to do here. 80 | return 81 | 82 | if upload is None: 83 | # File exists and we're just checking. 84 | message = "Available" 85 | else: 86 | # File exists and we wanted to upload it. Just say it's there already. 87 | upload.file.close() 88 | s = os.stat(path) 89 | message = "File %s already exists\nFile: %s Size: %d" % ( 90 | filename, path, s[stat.ST_SIZE]) 91 | send_ok(message) 92 | sys.exit(0) 93 | 94 | 95 | # Get correct hashing algorithm 96 | if 'sha512sum' in form: 97 | checksum = check_form('sha512sum') 98 | hash_type = 'sha512' 99 | elif 'md5sum' in form: 100 | checksum = check_form('md5sum') 101 | hash_type = 'md5' 102 | else: 103 | send_error('Required checksum is not present') 104 | 105 | 106 | NAME = check_form("name", strip=False) 107 | 108 | if '/' not in NAME: 109 | NAME = '%s/%s' % (DEFAULT_NAMESPACE, NAME) 110 | 111 | # Is this a submission or a test? 112 | FILE = None 113 | FILENAME = None 114 | if form.has_key("filename"): 115 | # check the presence of the file 116 | FILENAME = check_form("filename") 117 | else: 118 | if form.has_key("file"): 119 | FILE = form["file"] 120 | if not FILE.file: 121 | send_error("No file given for upload. Aborting") 122 | try: 123 | FILENAME = os.path.basename(FILE.filename) 124 | except: 125 | send_error("Could not extract the filename for upload. Aborting") 126 | else: 127 | send_error("required field '%s' is not present" % ("file", )) 128 | 129 | # Now that all the fields are valid,, figure out our operating environment 130 | if not os.environ.has_key("SCRIPT_FILENAME"): 131 | send_error("My running environment is funky. Aborting") 132 | 133 | # try to see if we already have this file... 134 | file_dest = "%s/%s/%s/%s/%s/%s" % (CACHE_DIR, NAME, FILENAME, hash_type, checksum, FILENAME) 135 | old_path = "%s/%s/%s/%s/%s" % (CACHE_DIR, NAME, FILENAME, checksum, FILENAME) 136 | 137 | check_file_exists(file_dest, FILENAME, FILE) 138 | if hash_type == 'md5': 139 | # If we're using MD5, handle the case of the file only existing in the old 140 | # path. Once the new checksum is fully deployed, this condition will become 141 | # obsolete. 142 | check_file_exists(old_path, FILENAME, FILE) 143 | 144 | # just checking? 145 | if FILE is None: 146 | send_ok("Missing") 147 | sys.exit(-9) 148 | 149 | # if a directory exists, check that it has the proper permissions 150 | def check_dir(tmpdir, wok=os.W_OK): 151 | if not os.access(tmpdir, os.F_OK): 152 | return 0 153 | if not os.access(tmpdir, os.R_OK|wok|os.X_OK): 154 | send_error("Unable to write to %s repository." % ( 155 | tmpdir,)) 156 | if not os.path.isdir(tmpdir): 157 | send_error("Path %s is not a directory." % (tmpdir,)) 158 | return 1 159 | 160 | if not os.environ.has_key("SCRIPT_FILENAME"): 161 | send_error("My running environment is funky. Aborting") 162 | # the module's top level directory 163 | my_moddir = "%s/%s" % (CACHE_DIR, NAME) 164 | my_filedir = "%s/%s" % (my_moddir, FILENAME) 165 | hash_dir = "%s/%s/%s" % (my_filedir, hash_type, checksum) 166 | 167 | # first test if the module really exists 168 | if not check_dir("%s/%s.git" % (GITREPO, NAME), 0): 169 | log_msg("Unknown module", NAME) 170 | send_ok("Module '%s' does not exist!" % (NAME,)) 171 | sys.exit(-9) 172 | 173 | makedirs(my_moddir) 174 | 175 | # grab a temporary filename and dump our file in there 176 | tempfile.tempdir = my_moddir 177 | tmpfile = tempfile.mktemp(checksum) 178 | tmpfd = open(tmpfile, "wb+") 179 | # now read the whole file in 180 | m = hashlib.new(hash_type) 181 | FILELENGTH=0 182 | while 1: 183 | s = FILE.file.read(BUFFER_SIZE) 184 | if not s: 185 | break 186 | tmpfd.write(s) 187 | m.update(s) 188 | FILELENGTH = FILELENGTH + len(s) 189 | # now we're done reading, check the MD5 sum of what we got 190 | tmpfd.close() 191 | check_checksum = m.hexdigest() 192 | if checksum != check_checksum: 193 | send_error("%s check failed. Received %s instead of %s" % ( 194 | hash_type.upper(), check_checksum, checksum)) 195 | # wow, even the checksum matches. make sure full path is valid now 196 | makedirs(hash_dir) 197 | # and move our file to the final location 198 | os.rename(tmpfile, file_dest) 199 | log_msg("Stored %s (%s bytes)" % (file_dest, FILELENGTH)) 200 | send_ok("File %s Size %d STORED OK" % (FILENAME, FILELENGTH)) 201 | 202 | # The file was uploaded with MD5, so we hardlink it to the old location 203 | # (without hash type). This is a temporary workaround to make sure old clients 204 | # can access files they uploaded. 205 | if hash_type == 'md5': 206 | hardlink(file_dest, old_path) 207 | 208 | sys.exit(0) 209 | -------------------------------------------------------------------------------- /dist-git/images/server-communication.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/release-engineering/dist-git/851f4d9500c148a81229fcba237c0ac687a4de40/dist-git/images/server-communication.png -------------------------------------------------------------------------------- /dist-git/images/storage.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/release-engineering/dist-git/851f4d9500c148a81229fcba237c0ac687a4de40/dist-git/images/storage.png -------------------------------------------------------------------------------- /dist-git/images/tutorial.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/release-engineering/dist-git/851f4d9500c148a81229fcba237c0ac687a4de40/dist-git/images/tutorial.png -------------------------------------------------------------------------------- /dist-git/scripts/dist-git/dist-git-gc: -------------------------------------------------------------------------------- 1 | #!/usr/bin/bash 2 | # Cleanup unnecessary files 3 | # The default DEPTH is 2 because the directory path is /var/lib/dist-git/git/rpms/ 4 | 5 | REPODIR=$(crudini --get /etc/dist-git/dist-git.conf dist-git gitroot_dir) 6 | DEPTH=$(crudini --get /etc/dist-git/dist-git.conf dist-git git_gc_depth) 7 | 8 | find "$REPODIR" -mindepth "$DEPTH" -maxdepth "$DEPTH" -type d | 9 | while read -r DIRECTORY; do 10 | (cd "$DIRECTORY" && git gc) 11 | done 12 | -------------------------------------------------------------------------------- /dist-git/scripts/dist-git/hooks/grok_update: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | umask 0002 4 | 5 | eval "$(crudini --format=sh --get /etc/dist-git/dist-git.conf dist-git)" 6 | /usr/bin/grok-manifest -m $gitroot_dir/manifest.js.gz -t $gitroot_dir -n `pwd` 7 | -------------------------------------------------------------------------------- /dist-git/scripts/dist-git/hooks/post-receive: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | readarray revs 4 | 5 | dirpath="$GIT_DIR/hooks/post-receive-chained.d" 6 | files=("$dirpath"/*) 7 | 8 | # pee redirects stdin to each of the post-receive hooks in place. 9 | if [[ -e "$files" || -L "$files" ]]; then 10 | /usr/bin/pee "$dirpath"/* 11 | fi 12 | -------------------------------------------------------------------------------- /dist-git/scripts/dist-git/mkbranch: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Create a new development branch for a package. 4 | 5 | umask 0002 6 | 7 | # Figure out the environment we're running in 8 | RUNDIR=$(cd "$(dirname "$0")" && pwd) 9 | 10 | eval "$(crudini --format=sh --get /etc/dist-git/dist-git.conf dist-git)" 11 | REPODIR=$gitroot_dir 12 | SRC_BRANCH="$default_branch" 13 | 14 | # check if a moron is driving me 15 | if [ ! -d $REPODIR ] ; then 16 | # we're not on the git server (this check is fragile) 17 | echo "ERROR: This script has to be run on the git server." 18 | echo "ERROR: Homer sez 'Duh'." 19 | exit -9 20 | fi 21 | 22 | # Local variables 23 | VERBOSE=0 24 | TEST= 25 | IGNORE= 26 | BRANCH="" 27 | PACKAGES="" 28 | 29 | Usage() { 30 | cat <] ... 33 | 34 | Creates a new branch for the list of s. 35 | The /$SRC_BRANCH suffix on branch names is assumed. 36 | 37 | Options: 38 | -n,--test Don't do nothing, only test 39 | -i,--ignore Ignore erroneous modules 40 | -h,--help This help message 41 | -v,--verbose Increase verbosity 42 | EOF 43 | } 44 | 45 | # parse the arguments 46 | while [ -n "$1" ] ; do 47 | case "$1" in 48 | -h | --help ) 49 | Usage 50 | exit 0 51 | ;; 52 | 53 | -v | --verbose ) 54 | VERBOSE=$(($VERBOSE + 1)) 55 | ;; 56 | 57 | -i | --ignore ) 58 | IGNORE="yes" 59 | ;; 60 | 61 | -n | --test ) 62 | TEST="yes" 63 | ;; 64 | 65 | -b | --branch ) 66 | shift 67 | BRANCH=$1/$SRC_BRANCH 68 | ;; 69 | 70 | * ) 71 | if [ -z "$BRANCH" ] ; then 72 | BRANCH="$1" 73 | else 74 | PACKAGES="$PACKAGES $1" 75 | fi 76 | ;; 77 | esac 78 | shift 79 | done 80 | 81 | # check the arguments 82 | if [ -z "$BRANCH" -o -z "$PACKAGES" ] ; then 83 | Usage 84 | exit -1 85 | fi 86 | 87 | # prepend default namespace if set 88 | NEWP= 89 | for PACKAGE in $PACKAGES ; do 90 | PACKAGE=`echo $PACKAGE | sed -e "s+^/*\([^/]*\)/*$+\1+"` 91 | parts=($(echo $PACKAGE | tr "/" " ")) 92 | parts_len=${#parts[@]} 93 | if [ -n "$default_namespace" ] && [ $parts_len -le 1 ]; then 94 | PACKAGE=$default_namespace/$PACKAGE 95 | fi 96 | NEWP="$NEWP $PACKAGE" 97 | done 98 | PACKAGES="$(echo $NEWP)" 99 | 100 | # Sanity checks before we start doing damage 101 | NEWP= 102 | for p in $PACKAGES ; do 103 | [ $VERBOSE -gt 1 ] && echo "Checking package $p..." 104 | if [ ! -d $REPODIR/$p.git ] ; then 105 | echo "ERROR: Package module $p is invalid" >&2 106 | [ "$IGNORE" = "yes" ] && continue || exit -1 107 | fi 108 | if GIT_DIR=$REPODIR/$p.git git rev-parse -q --verify $BRANCH >/dev/null; then 109 | echo "IGNORING: Package module $p already has a branch $BRANCH" >&2; 110 | [ "$IGNORE" = "yes" ] && continue || exit 128 111 | fi 112 | NEWP="$NEWP $p" 113 | done 114 | PACKAGES="$(echo $NEWP)" 115 | 116 | if [ -z "$PACKAGES" ] ; then 117 | echo "NOOP: no valid packages found to process" 118 | exit -1 119 | fi 120 | 121 | if [ -n "$TEST" ] ; then 122 | echo "Branch $BRANCH valid for $PACKAGES" 123 | exit 0 124 | fi 125 | 126 | # "global" permissions check 127 | if [ ! -w $REPODIR ] ; then 128 | echo "ERROR: You can not write to $REPODIR" 129 | echo "ERROR: You can not perform branching operations" 130 | exit -1 131 | fi 132 | 133 | # Now start working on creating those branches 134 | 135 | # For every module, "create" the branch 136 | for NAME in $PACKAGES ; do 137 | echo 138 | echo "Creating new module branch '$BRANCH' for '$NAME'..." 139 | 140 | # permissions checks for this particular module 141 | if [ ! -w $REPODIR/$NAME.git/refs/heads/ ] ; then 142 | echo "ERROR: You can not write to $d" 143 | echo "ERROR: $NAME can not be branched by you" 144 | continue 145 | fi 146 | [ $VERBOSE -gt 0 ] && echo "Creating $NAME $BRANCH from $NAME ..." 147 | $(pushd $REPODIR/$NAME.git >/dev/null && \ 148 | git branch --no-track $BRANCH `git rev-list --max-parents=0 "$SRC_BRANCH" | head -1` && \ 149 | popd >/dev/null) || { 150 | echo "ERROR: Branch $NAME $BRANCH could not be created" >&2 151 | popd >/dev/null 152 | exit -2 153 | } 154 | if [[ $grok && $grok != "False" ]]; then 155 | /usr/bin/grok-manifest -m $REPODIR/manifest.js.gz -t $REPODIR -n $REPODIR/$NAME.git 156 | fi 157 | done 158 | 159 | echo 160 | echo "Done." 161 | -------------------------------------------------------------------------------- /dist-git/scripts/dist-git/mkbranch_branching: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Create a new development branch for a package. 4 | 5 | umask 0002 6 | 7 | # Figure out the environment we're running in 8 | RUNDIR=$(cd "$(dirname "$0")" && pwd) 9 | 10 | eval "$(crudini --format=sh --get /etc/dist-git/dist-git.conf dist-git)" 11 | REPODIR=$gitroot_dir 12 | SRC_BRANCH="$default_branch" 13 | 14 | # check if a moron is driving me 15 | if [ ! -d $REPODIR ] ; then 16 | # we're not on the git server (this check is fragile) 17 | echo "ERROR: This script has to be run on the git server." 18 | echo "ERROR: Homer sez 'Duh'." 19 | exit -9 20 | fi 21 | 22 | # Local variables 23 | VERBOSE=0 24 | TEST= 25 | IGNORE= 26 | BRANCH="" 27 | PACKAGES="" 28 | 29 | Usage() { 30 | cat <] ... 33 | 34 | Creates a new branch for the list of s. 35 | The /$SRC_BRANCH suffix on branch names is assumed. 36 | 37 | Options: 38 | -n,--test Don't do nothing, only test 39 | -i,--ignore Ignore erroneous modules 40 | -h,--help This help message 41 | -v,--verbose Increase verbosity 42 | EOF 43 | } 44 | 45 | # parse the arguments 46 | while [ -n "$1" ] ; do 47 | case "$1" in 48 | -h | --help ) 49 | Usage 50 | exit 0 51 | ;; 52 | 53 | -v | --verbose ) 54 | VERBOSE=$(($VERBOSE + 1)) 55 | ;; 56 | 57 | -i | --ignore ) 58 | IGNORE="yes" 59 | ;; 60 | 61 | -n | --test ) 62 | TEST="yes" 63 | ;; 64 | 65 | -b | --branch ) 66 | shift 67 | BRANCH=$1/$SRC_BRANCH 68 | ;; 69 | 70 | * ) 71 | if [ -z "$BRANCH" ] ; then 72 | BRANCH="$1" 73 | else 74 | PACKAGES="$PACKAGES $1" 75 | fi 76 | ;; 77 | esac 78 | shift 79 | done 80 | 81 | # check the arguments 82 | if [ -z "$BRANCH" -o -z "$PACKAGES" ] ; then 83 | Usage 84 | exit -1 85 | fi 86 | 87 | # prepend default namespace if set 88 | NEWP= 89 | for PACKAGE in $PACKAGES ; do 90 | PACKAGE=`echo $PACKAGE | sed -e "s+^/*\([^/]*\)/*$+\1+"` 91 | parts=($(echo $PACKAGE | tr "/" " ")) 92 | parts_len=${#parts[@]} 93 | if [ -n "$default_namespace" ] && [ $parts_len -le 1 ]; then 94 | PACKAGE=$default_namespace/$PACKAGE 95 | fi 96 | NEWP="$NEWP $PACKAGE" 97 | done 98 | PACKAGES="$(echo $NEWP)" 99 | 100 | # Sanity checks before we start doing damage 101 | NEWP= 102 | for p in $PACKAGES ; do 103 | [ $VERBOSE -gt 1 ] && echo "Checking package $p..." 104 | if [ ! -d $REPODIR/$p.git ] ; then 105 | echo "ERROR: Package module $p is invalid" >&2 106 | [ "$IGNORE" = "yes" ] && continue || exit -1 107 | fi 108 | if GIT_DIR=$REPODIR/$p.git git rev-parse -q --verify $BRANCH >/dev/null; then 109 | echo "IGNORING: Package module $p already has a branch $BRANCH" >&2; 110 | [ "$IGNORE" = "yes" ] && continue || exit 128 111 | fi 112 | NEWP="$NEWP $p" 113 | done 114 | PACKAGES="$(echo $NEWP)" 115 | 116 | if [ -z "$PACKAGES" ] ; then 117 | echo "NOOP: no valid packages found to process" 118 | exit -1 119 | fi 120 | 121 | if [ -n "$TEST" ] ; then 122 | echo "Branch $BRANCH valid for $PACKAGES" 123 | exit 0 124 | fi 125 | 126 | # "global" permissions check 127 | if [ ! -w $REPODIR ] ; then 128 | echo "ERROR: You can not write to $REPODIR" 129 | echo "ERROR: You can not perform branching operations" 130 | exit -1 131 | fi 132 | 133 | # Now start working on creating those branches 134 | 135 | # For every module, "create" the branch 136 | for NAME in $PACKAGES ; do 137 | echo 138 | echo "Creating new module branch '$BRANCH' for '$NAME'..." 139 | 140 | # permissions checks for this particular module 141 | if [ ! -w $REPODIR/$NAME.git/refs/heads/ ] ; then 142 | echo "ERROR: You can not write to $d" 143 | echo "ERROR: $NAME can not be branched by you" 144 | continue 145 | fi 146 | [ $VERBOSE -gt 0 ] && echo "Creating $NAME $BRANCH from $NAME ..." 147 | $(pushd $REPODIR/$NAME.git >/dev/null && \ 148 | git branch --no-track $BRANCH `git rev-list "$SRC_BRANCH" | head -1` && \ 149 | popd >/dev/null) || { 150 | echo "ERROR: Branch $NAME $BRANCH could not be created" >&2 151 | popd >/dev/null 152 | exit -2 153 | } 154 | if [[ $grok && $grok != "False" ]]; then 155 | /usr/bin/grok-manifest -m $REPODIR/manifest.js.gz -t $REPODIR -n $REPODIR/$NAME.git 156 | fi 157 | done 158 | 159 | echo 160 | echo "Done." 161 | -------------------------------------------------------------------------------- /dist-git/scripts/dist-git/remove_unused_sources: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | Usage() { 4 | cat < 7 | 8 | Removes all tarballs in except 9 | for tarballs that are referenced by the latest commit of each 10 | branch in . 11 | EOF 12 | } 13 | 14 | die() { echo "$*" 1>&2 ; exit 1; } 15 | 16 | logcmd () 17 | { 18 | echo >&2 "[PID$$:DATE:$(date +"%Y%m%d_%H%M%S")] $*" 19 | "$@" 20 | } 21 | 22 | if [[ $# != 2 ]]; then 23 | Usage 24 | exit 1 25 | fi 26 | 27 | pkg_git_dir="$1" 28 | pkg_lookaside_dir="$2" 29 | 30 | if [ ! -d "$pkg_git_dir" ]; then 31 | echo "$pkg_git_dir is not a valid directory." 32 | exit 1 33 | fi 34 | if [ ! -d "$pkg_lookaside_dir" ]; then 35 | echo "$pkg_lookaside_dir is not a valid directory." 36 | exit 1 37 | fi 38 | 39 | pushd "$pkg_git_dir" > /dev/null || exit 1 40 | 41 | allowlist=() 42 | 43 | # find sources that are referenced by the latest commit in any of the branches 44 | for branch in $(git for-each-ref --format="%(refname:short)" refs/heads); do 45 | while read -r line; do 46 | set -- $line 47 | hash=$1 48 | filename=$2 49 | # skip projects using the new format 50 | test $# -eq 0 && continue 51 | test $# -ne 2 && die "Unsupported format. Only the old ' 27 | 28 | Creates a new repo for 29 | 30 | Options: 31 | -h,--help This help message 32 | EOF 33 | } 34 | 35 | while [[ $# > 0 ]] 36 | do 37 | key="$1" 38 | case $key in 39 | --help|-h) 40 | Usage 41 | exit 0 42 | ;; 43 | *) 44 | PACKAGE="$1" 45 | ;; 46 | esac 47 | shift 48 | done 49 | 50 | # check the arguments 51 | if [ -z "$PACKAGE" ] ; then 52 | Usage 53 | exit -1 54 | fi 55 | 56 | # optionally prepend default namespace if $PACKAGE is not namespaced 57 | PACKAGE=`echo $PACKAGE | sed -e "s+^/*\([^/]*\)/*$+\1+"` 58 | parts=($(echo $PACKAGE | tr "/" " ")) 59 | parts_len=${#parts[@]} 60 | if [ -n "$default_namespace" ] && [ $parts_len -le 1 ]; then 61 | PACKAGE=$default_namespace/$PACKAGE 62 | fi 63 | 64 | # Sanity checks before we start doing damage 65 | [ $VERBOSE -gt 1 ] && echo "Checking package $PACKAGE..." 66 | if [ -f $REPODIR/$PACKAGE.git/refs/heads/$DEFAULT_BRANCH ] ; then 67 | echo "ERROR: Package module $PACKAGE already exists!" >&2 68 | exit 128 69 | fi 70 | 71 | # "global" permissions check 72 | if [ ! -w $REPODIR ] ; then 73 | echo "ERROR: You can not write to $REPODIR" 74 | echo "ERROR: You can not create repos" 75 | exit -1 76 | fi 77 | 78 | # Now start working on creating those branches 79 | # Create a tmpdir to do some git work in 80 | TMPDIR=$(mktemp -d /tmp/tmpXXXXXX) 81 | 82 | # First create the master repo 83 | mkdir -p $REPODIR/$PACKAGE.git 84 | pushd $REPODIR/$PACKAGE.git >/dev/null 85 | git init -q --shared --bare 86 | popd >/dev/null 87 | 88 | mkdir -p $REPODIR/$PACKAGE.git/hooks/post-receive-chained.d 89 | if [[ $grok && $grok != "False" ]]; then 90 | if ! [ -f $REPODIR/manifest.js.gz ]; then 91 | echo 'Generating initial grok manifest...' 92 | /usr/bin/grok-manifest -m $REPODIR/manifest.js.gz -t $REPODIR 93 | fi 94 | ln -s /usr/share/dist-git/hooks/grok_update \ 95 | $REPODIR/$PACKAGE.git/hooks/post-receive-chained.d/grok_update 96 | fi 97 | # This one kicks off all the others in post-receive-chained.d 98 | ln -s /usr/share/dist-git/hooks/post-receive \ 99 | $REPODIR/$PACKAGE.git/hooks/post-receive 100 | 101 | # Now clone that repo and create the .gitignore and sources file 102 | git init -q $TMPDIR/$PACKAGE 103 | pushd $TMPDIR/$PACKAGE >/dev/null 104 | touch .gitignore 105 | git config user.name "$git_author_name" 106 | git config user.email "$git_author_email" 107 | git add . 108 | git commit -q -m 'Initial setup of the repo' 109 | git remote add origin $REPODIR/$PACKAGE.git 110 | git push -q origin "$DEFAULT_BRANCH" 111 | popd >/dev/null 112 | 113 | # Place the gitolite update hook in place since we're not using our own 114 | if [[ $gitolite && $gitolite != "False" ]]; then 115 | ln -s /etc/gitolite/hooks/common/update $REPODIR/$PACKAGE.git/hooks/update 116 | fi 117 | 118 | rm -rf $TMPDIR 119 | echo "Done." 120 | -------------------------------------------------------------------------------- /dist-git/scripts/httpd/upload.cgi: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | # 3 | # CGI script to handle file updates for the rpms git repository. There 4 | # is nothing really complex here other than tedious checking of our 5 | # every step along the way... 6 | # 7 | # License: GPL 8 | 9 | import cgi 10 | import errno 11 | import grp 12 | import hashlib 13 | import os 14 | import sys 15 | import tempfile 16 | import time 17 | 18 | from configparser import ConfigParser 19 | 20 | # Reading buffer size 21 | BUFFER_SIZE = 4096 22 | 23 | # Fedora Packager Group 24 | PACKAGER_GROUP = 'packager' 25 | 26 | # Path to a config file 27 | CONFIG = os.environ.get('DISTGIT_CONFIG', '/etc/dist-git/dist-git.conf') 28 | 29 | 30 | def send_error(text, status='500 Internal Server Error'): 31 | """Send an error back to the client 32 | 33 | This ensures that the client will get a proper error, including the HTTP 34 | status code, so that it can handle problems appropriately. 35 | 36 | Args: 37 | text (str): The error message to send the client 38 | status (str, optional): The HTTP status code to return to the client. 39 | """ 40 | print('Status: %s' % status) 41 | print('Content-type: text/plain\n') 42 | print(text) 43 | 44 | sys.exit(0) 45 | 46 | 47 | def send(text, exit=True): 48 | """Send a success message back to the client 49 | 50 | Args: 51 | text (str): The message to send the client 52 | exit (bool, optional): If we should exit immediatelly or not. 53 | Use this if you want to additionally print out more content 54 | into response. 55 | """ 56 | print('Status: 200 OK') 57 | print('Content-type: text/plain\n') 58 | print(text) 59 | 60 | if exit: 61 | sys.exit(0) 62 | 63 | 64 | def check_form(form, var): 65 | ret = form.getvalue(var, None) 66 | 67 | if ret is None: 68 | send_error('Required field "%s" is not present.' % var, 69 | status='400 Bad Request') 70 | 71 | if isinstance(ret, list): 72 | send_error('Multiple values given for "%s". Aborting.' % var, 73 | status='400 Bad Request') 74 | return ret 75 | 76 | 77 | def check_group(username): 78 | authenticated = False 79 | 80 | try: 81 | if username in grp.getgrnam(PACKAGER_GROUP)[3]: 82 | authenticated = True 83 | except KeyError: 84 | pass 85 | 86 | return authenticated 87 | 88 | 89 | def hardlink(src, dest, username): 90 | makedirs(os.path.dirname(dest), username) 91 | 92 | try: 93 | os.link(src, dest) 94 | except OSError as e: 95 | if e.errno != errno.EEXIST: 96 | send_error(str(e)) 97 | 98 | # The file already existed at the dest path, hardlink over it 99 | os.unlink(dest) 100 | os.link(src, dest) 101 | 102 | sys.stderr.write("[username=%s] ln %s %s\n" % (username, src, dest)) 103 | 104 | 105 | def makedirs(dir_, username, mode=0o2755): 106 | try: 107 | os.makedirs(dir_, mode=mode) 108 | sys.stderr.write('[username=%s] mkdir %s\n' % (username, dir_)) 109 | except OSError as e: 110 | if e.errno != errno.EEXIST: 111 | send_error(str(e)) 112 | 113 | 114 | def ensure_namespaced(name, namespace): 115 | if not namespace: 116 | return name 117 | 118 | name_parts = name.split('/') 119 | if len(name_parts) == 1: 120 | return os.path.join(namespace, name) 121 | 122 | return name 123 | 124 | 125 | def get_checksum_and_hash_type(form): 126 | # Search for the file hash, start with stronger hash functions 127 | if 'sha512sum' in form: 128 | checksum = check_form(form, 'sha512sum') 129 | hash_type = "sha512" 130 | 131 | elif 'md5sum' in form: 132 | # Fallback on md5 133 | checksum = check_form(form, 'md5sum') 134 | hash_type = "md5" 135 | 136 | else: 137 | send_error('Required checksum is not present', 138 | status='400 Bad Request') 139 | 140 | return checksum, hash_type 141 | 142 | 143 | def emit_message(config, name, checksum, filename, username, msgpath): 144 | emit_fedmsg(config, name, checksum, filename, username, msgpath) 145 | emit_fedora_message(config, name, checksum, filename, username, msgpath) 146 | 147 | 148 | def emit_fedmsg(config, name, checksum, filename, username, msgpath): 149 | # Emit a fedmsg message. Load the config to talk to the fedmsg-relay. 150 | if config.getboolean('upload', 'fedmsgs', fallback=True): 151 | try: 152 | import fedmsg 153 | import fedmsg.config 154 | 155 | config = fedmsg.config.load_config([], None) 156 | config['active'] = True 157 | config['endpoints']['relay_inbound'] = config['relay_inbound'] 158 | fedmsg.init(name="relay_inbound", cert_prefix="lookaside", **config) 159 | 160 | topic = "lookaside.new" 161 | msg = dict(name=name, md5sum=checksum, 162 | filename=filename.split('/')[-1], agent=username, 163 | path=msgpath) 164 | fedmsg.publish(modname="git", topic=topic, msg=msg) 165 | except Exception as e: 166 | sys.stderr.write("Error with fedmsg", str(e)) 167 | 168 | 169 | def emit_fedora_message(config, name, checksum, filename, username, msgpath): 170 | # Emit a fedmsg-messaging message 171 | if not config.getboolean('upload', 'fedora_messaging', fallback=True): 172 | return 173 | try: 174 | import fedora_messaging.api 175 | import fedora_messaging.config 176 | import fedora_messaging.exceptions 177 | except ImportError: 178 | sys.stderr.write( 179 | "fedora-messaging must be installed for the notifications to work.") 180 | return 181 | 182 | try: 183 | if config['upload'].get('fedora_messaging_config'): 184 | fedora_messaging.config.conf.load_config( 185 | config['upload']['fedora_messaging_config']) 186 | 187 | message = dict( 188 | name=name, 189 | md5sum=checksum, 190 | filename=filename.split('/')[-1], 191 | agent=username, 192 | path=msgpath 193 | ) 194 | 195 | msg = fedora_messaging.api.Message( 196 | topic="git.lookaside.new", 197 | body=message 198 | ) 199 | fedora_messaging.api.publish(msg) 200 | except fedora_messaging.exceptions.PublishReturned as e: 201 | sys.stderr.write("Fedora Messaging broker rejected message %s: %s" % (msg.id, e)) 202 | except fedora_messaging.exceptions.ConnectionException as e: 203 | sys.stderr.write("Error sending message %s: %s" % (msg.id, e)) 204 | except Exception as e: 205 | sys.stderr.write("Error sending fedora-messaging message.") 206 | sys.stderr.write("ERROR: %s\n" % e) 207 | 208 | 209 | def get_config(): 210 | config = ConfigParser() 211 | config.read(CONFIG) 212 | return config 213 | 214 | 215 | def main(): 216 | form = cgi.FieldStorage() 217 | config = get_config() 218 | os.umask(0o002) 219 | 220 | username = os.environ.get('SSL_CLIENT_S_DN_CN', None) 221 | gssname = os.environ.get('GSS_NAME', os.environ.get('REMOTE_USER', None)) 222 | if gssname and '@' in gssname and not username: 223 | username = gssname.partition('@')[0] 224 | 225 | if not config.getboolean('upload', 'disable_group_check', fallback=False) and\ 226 | not check_group(username): 227 | send_error('You must connect with a valid certificate and be in the ' 228 | '%s group to upload.' % PACKAGER_GROUP, 229 | status='403 Forbidden') 230 | 231 | assert os.environ['REQUEST_URI'].split('/')[1] == 'repo' 232 | 233 | name = check_form(form, 'name').strip('/') 234 | checksum, hash_type = get_checksum_and_hash_type(form) 235 | 236 | action = None 237 | upload_file = None 238 | filename = None 239 | 240 | # Is this a submission or a test? 241 | # in a test, we don't get a file, just a filename. 242 | # In a submission, we don't get a filename, just the file. 243 | if 'filename' in form: 244 | action = 'check' 245 | filename = check_form(form, 'filename') 246 | filename = os.path.basename(filename) 247 | sys.stderr.write('[username=%s] Checking file status: NAME=%s ' 248 | 'FILENAME=%s %sSUM=%s\n' % (username, name, filename, 249 | hash_type.upper(), 250 | checksum)) 251 | else: 252 | action = 'upload' 253 | if 'file' in form: 254 | upload_file = form['file'] 255 | if not upload_file.file: 256 | send_error('No file given for upload. Aborting.', 257 | status='400 Bad Request') 258 | filename = os.path.basename(upload_file.filename) 259 | else: 260 | send_error('Required field "file" is not present.', 261 | status='400 Bad Request') 262 | 263 | sys.stderr.write('[username=%s] Processing upload request: ' 264 | 'NAME=%s FILENAME=%s %sSUM=%s\n' % ( 265 | username, name, filename, hash_type.upper(), 266 | checksum)) 267 | 268 | # prefix name by default namespace if configured 269 | if config['dist-git'].get('default_namespace'): 270 | name = ensure_namespaced(name, config['dist-git'].get('default_namespace')).strip('/') 271 | 272 | if config['dist-git'].get('lookaside_dir'): 273 | module_dir = os.path.join(config['dist-git']['lookaside_dir'], name) 274 | elif config['dist-git'].get('cache_dir'): # deprecated 275 | module_dir = os.path.join(config['dist-git']['cache_dir'], 'lookaside/pkgs', name) 276 | else: 277 | raise Exception('Please, set lookaside_dir config option.') 278 | 279 | hash_dir = os.path.join(module_dir, filename, hash_type, checksum) 280 | msgpath = os.path.join(name, filename, hash_type, checksum, filename) 281 | 282 | # first test if the module really exists 283 | git_dir = os.path.join(config['dist-git']['gitroot_dir'], '%s.git' % name) 284 | if not os.path.isdir(git_dir): 285 | sys.stderr.write('[username=%s] Unknown module: %s' % (username, name)) 286 | send_error('Module "%s" does not exist!' % name, 287 | status='404 Not Found') 288 | 289 | # try to see if we already have this file... 290 | dest_file = os.path.join(hash_dir, filename) 291 | old_dir = os.path.join(module_dir, filename, checksum) 292 | old_path = os.path.join(old_dir, filename) 293 | 294 | if os.path.exists(dest_file): 295 | if action == 'check': 296 | send('Available') 297 | else: 298 | upload_file.file.close() 299 | dest_file_stat = os.stat(dest_file) 300 | msg = 'File %s already exists\n' % filename 301 | msg += 'File: %s Size: %d' % (dest_file, dest_file_stat.st_size) 302 | send(msg) 303 | 304 | elif action == 'check': 305 | if os.path.exists(old_path): 306 | # The file had been uploaded at the old path 307 | hardlink(old_path, dest_file, username) 308 | send('Available') 309 | else: 310 | send('Missing') 311 | 312 | elif hash_type == "md5" and config.getboolean('upload', 'nomd5', fallback=True): 313 | send_error('Uploads with md5 are no longer allowed.', 314 | status='406 Not Acceptable') 315 | 316 | # check that all directories are in place 317 | makedirs(module_dir, username) 318 | 319 | # grab a temporary filename and dump our file in there 320 | tempfile.tempdir = module_dir 321 | tmpfile = tempfile.mkstemp(checksum)[1] 322 | tmpfd = open(tmpfile, 'wb') 323 | 324 | # now read the whole file in 325 | m = getattr(hashlib, hash_type)() 326 | filesize = 0 327 | while True: 328 | data = upload_file.file.read(BUFFER_SIZE) 329 | if not data: 330 | break 331 | tmpfd.write(data) 332 | m.update(data) 333 | filesize += len(data) 334 | 335 | # now we're done reading, check the checksum of what we got 336 | tmpfd.close() 337 | check_checksum = m.hexdigest() 338 | if checksum != check_checksum: 339 | os.unlink(tmpfile) 340 | send_error("%s check failed. Received %s instead of %s." % 341 | (hash_type.upper(), check_checksum, checksum), 342 | status='400 Bad Request') 343 | 344 | # wow, even the checksum matches. make sure full path is valid now 345 | makedirs(hash_dir, username) 346 | os.rename(tmpfile, dest_file) 347 | os.chmod(dest_file, 0o644) 348 | 349 | # set mtime of the uploaded file if provided 350 | if 'mtime' in form: 351 | mtime_str = form.getvalue('mtime') 352 | try: 353 | mtime = float(mtime_str) 354 | except ValueError: 355 | send_error('Invalid value sent for mtime "%s". Aborting.' % mtime_str, 356 | status='400 Bad Request') 357 | 358 | os.utime(dest_file, (time.time(), mtime)) 359 | 360 | sys.stderr.write('[username=%s] Stored %s (%d bytes)' % (username, 361 | dest_file, 362 | filesize)) 363 | send('File %s size %d %s %s stored OK' % (filename, filesize, 364 | hash_type.upper(), checksum), exit=False) 365 | 366 | # Add the file to the old path, where fedpkg used to look for 367 | if hash_type == "md5" and config.getboolean('upload', 'old_paths', fallback=True): 368 | hardlink(dest_file, old_path, username) 369 | 370 | emit_message(config, name, checksum, filename, username, msgpath) 371 | 372 | 373 | if __name__ == '__main__': 374 | try: 375 | main() 376 | except Exception as e: 377 | import traceback 378 | sys.stderr.write('%s\n' % traceback.format_exc()) 379 | send_error(str(e)) 380 | -------------------------------------------------------------------------------- /dist-git/selinux/dist_git.fc: -------------------------------------------------------------------------------- 1 | /var/lib/dist-git/git(/.*)? gen_context(system_u:object_r:git_user_content_t,s0) 2 | /var/lib/dist-git/cache(/.*)? gen_context(system_u:object_r:git_rw_content_t,s0) 3 | /var/lib/dist-git/web/upload.cgi gen_context(system_u:object_r:git_script_exec_t,s0) 4 | -------------------------------------------------------------------------------- /dist-git/selinux/dist_git.if: -------------------------------------------------------------------------------- 1 | 2 | # this file is not in use 3 | -------------------------------------------------------------------------------- /dist-git/selinux/dist_git.te: -------------------------------------------------------------------------------- 1 | policy_module(dist_git,1.0.1) 2 | 3 | 4 | require { 5 | type httpd_git_script_t; 6 | type git_script_tmp_t; 7 | type git_system_t; 8 | type git_user_content_t; 9 | type httpd_t; 10 | } 11 | 12 | files_tmp_file(git_script_tmp_t); 13 | allow httpd_git_script_t git_script_tmp_t:file manage_file_perms; 14 | 15 | # List the contents of the sysfs directories. 16 | dev_list_sysfs(httpd_git_script_t); 17 | 18 | # Allow sending logs to syslog 19 | logging_send_syslog_msg(httpd_git_script_t); 20 | 21 | # Get the attributes of all pty device nodes. 22 | term_getattr_all_ptys(httpd_git_script_t); 23 | 24 | # Get the attributes of all tty device nodes. 25 | term_getattr_all_ttys(httpd_git_script_t); 26 | 27 | # Do not audit attempts to get the attributes of generic pty devices. 28 | term_dontaudit_getattr_generic_ptys(httpd_git_script_t); 29 | 30 | # For git-daemon 31 | allow git_system_t git_user_content_t:dir { search getattr open read }; 32 | allow git_system_t git_user_content_t:file { read open getattr }; 33 | allow git_system_t git_user_content_t:lnk_file { read open getattr }; 34 | optional_policy(` 35 | gen_require(` class file map; ') 36 | allow git_system_t git_user_content_t:file map; 37 | ') 38 | 39 | # For git-http-backend 40 | allow httpd_t git_user_content_t:dir { search getattr open read }; 41 | allow httpd_t git_user_content_t:file { read open getattr }; 42 | allow httpd_t git_user_content_t:lnk_file { read open getattr }; 43 | optional_policy(` 44 | gen_require(` class file map; ') 45 | allow httpd_t git_user_content_t:file map; 46 | ') 47 | -------------------------------------------------------------------------------- /dist-git/tests/test_upload_script.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from __future__ import print_function 4 | 5 | import errno 6 | import os 7 | try: 8 | import unittest2 as unittest 9 | except ImportError: 10 | import unittest 11 | import tempfile 12 | import shutil 13 | import re 14 | import subprocess 15 | import requests 16 | import time 17 | import random 18 | from configparser import ConfigParser 19 | 20 | from parameterized import parameterized 21 | 22 | 23 | # Path to the actual CGI script that should be tested 24 | CGI_SCRIPT = os.path.join(os.path.dirname(__file__), '../scripts/httpd/upload.cgi') 25 | 26 | # A snippet for creating the server in a temporary location. We need to write a 27 | # separate script as it needs to run with working directory set to the 28 | # temporary directory. 29 | SERVER = """#!/usr/bin/{python} 30 | from {http_package} import HTTPServer 31 | from {cgi_package} import CGIHTTPRequestHandler 32 | s = HTTPServer(('%s', %s), CGIHTTPRequestHandler) 33 | s.handle_request() 34 | """.format(python="python3", 35 | http_package="http.server", 36 | cgi_package="http.server") 37 | 38 | # MD5 hash of "hello.txt" and "new.txt" strings used in a few tests 39 | HASH = '2e54144ba487ae25d03a3caba233da71' 40 | NEW_HASH = 'fce67ea4590d3b789fff55a37271f29f' 41 | SHA512 = 'acec329f80cc50edbab0dfbc2283d427ac673f84e6d8b949101791867b9b7771a53d2ffb1f8386189227beed4395b9a78171a1349700e2885c70ae14358d72ff' # noqa 42 | NEW_SHA512 = 'd3d67bc3e3848925892de9b132c9ff4054a05c9dbc7b4366d16b5c6b87c898df60da162e9ec415d4dc16470128ef52c11c44fc06da2841543ddeb351b10e9fb2' # noqa 43 | 44 | # The first value is what will be sent in the request, the second is the full 45 | # namespaced module name. 46 | EXISTING_MODULES = [ 47 | ("pkg", "rpms/pkg"), 48 | ("rpms/pkg", "rpms/pkg"), 49 | ("apbs/pkg", "apbs/pkg"), 50 | ] 51 | NON_EXISTING_MODULES = [ 52 | ("bad", "rpms/bad"), 53 | ("rpms/bad", "rpms/bad"), 54 | ("apbs/bad", "apbs/bad"), 55 | ] 56 | OLD_FILE_MODULES = [ 57 | ('old', 'rpms/old'), 58 | ('rpms/old', 'rpms/old'), 59 | ('apbs/old', 'apbs/old'), 60 | ] 61 | 62 | GIT_DIR = 'srv/git' 63 | CACHE_DIR = 'srv/cache/lookaside' 64 | 65 | 66 | class UploadTest(unittest.TestCase): 67 | 68 | def setUp(self): 69 | self.hostname = 'localhost' 70 | # Koji can run multiple builds on the same host in parallel 71 | self.port = random.randrange(10000, 20000, step=1) 72 | # Create temporary filesystem tree 73 | self.topdir = tempfile.mkdtemp() 74 | os.chmod(self.topdir, 0o0777) 75 | # Copy cgi script and tweak it with new path 76 | cgi = os.path.join(self.topdir, 'cgi-bin', 'upload.cgi') 77 | os.mkdir(os.path.join(self.topdir, 'cgi-bin')) 78 | _copy_tweak(CGI_SCRIPT, cgi, self.topdir) 79 | shutil.copystat(CGI_SCRIPT, cgi) 80 | # Generate temporary distgit config for this test run 81 | self.config = _dump_config_file(self.topdir) 82 | 83 | self._run_server() 84 | 85 | # Create a package with a single source file in each namespace 86 | for _, module in EXISTING_MODULES: 87 | self.setup_module(module) 88 | self.touch('%s/%s/hello.txt/md5/%s/hello.txt' % (CACHE_DIR, module, HASH)) 89 | self.touch('%s/%s/hello.txt/sha512/%s/hello.txt' % (CACHE_DIR, module, SHA512)) 90 | 91 | # These are modules with sources in old MD5 path only. 92 | for _, module in OLD_FILE_MODULES: 93 | self.setup_module(module) 94 | self.touch('%s/%s/hello.txt/%s/hello.txt' % (CACHE_DIR, module, HASH)) 95 | 96 | def tearDown(self): 97 | shutil.rmtree(self.topdir) 98 | if not self.server.poll(): 99 | # The server did not exit yet, so let's kill it. 100 | try: 101 | self.server.terminate() 102 | except OSError as exc: 103 | # It's possible for the server to exit before we try to kill 104 | # it. In that case we get ESRCH (No such process). This is 105 | # fine. All other exceptions should still be reported and fail 106 | # the tests. 107 | if exc.errno != errno.ESRCH: 108 | raise 109 | self.server.wait() 110 | 111 | def _log_output(self): 112 | self.output.seek(0) 113 | print(self.output.read()) 114 | 115 | def _run_server(self): 116 | """Start a server in a temporary directory, and capture its output.""" 117 | script = os.path.join(self.topdir, 'server.py') 118 | with open(script, 'w') as f: 119 | f.write(SERVER % (self.hostname, self.port)) 120 | os.chmod(script, 0o0755) 121 | self.output = tempfile.TemporaryFile() 122 | self.server = subprocess.Popen(script, cwd=self.topdir, 123 | stdout=self.output, 124 | stderr=subprocess.STDOUT, 125 | env={'SCRIPT_FILENAME': 'foo', 126 | 'DISTGIT_CONFIG': self.config, 127 | 'REQUEST_URI': '/repo/foo/bar/upload.cgi'}) 128 | time.sleep(0.1) # Wait for server to be up. 129 | self.url = 'http://%s:%s/cgi-bin/upload.cgi' % (self.hostname, self.port) 130 | 131 | def upload(self, name, hash, hashtype='md5', filename=None, filepath=None, mtime=None): 132 | """Send a request to the CGI script. Exactly one of filename and 133 | filepath has to be provided. 134 | 135 | :param name: name of the module 136 | :param hash: hash of the file 137 | :param filename: name of a file to check 138 | :param filepath: path to a file to upload 139 | """ 140 | args = { 141 | 'name': name, 142 | '%ssum' % hashtype: hash, 143 | } 144 | if filename: 145 | args['filename'] = filename 146 | if mtime: 147 | args["mtime"] = mtime 148 | 149 | files = None 150 | if filepath: 151 | files = {'file': open(filepath, 'rb')} 152 | 153 | response = requests.post(self.url, data=args, files=files) 154 | self._log_output() 155 | self.assertEqual(response.status_code, 200) 156 | return response.text 157 | 158 | def touch(self, filename, contents=None): 159 | """Create a file in a given location and return its path.""" 160 | contents = contents or filename 161 | path = os.path.join(self.topdir, filename) 162 | try: 163 | os.makedirs(os.path.dirname(path)) 164 | except OSError: 165 | pass 166 | print('Creating %s' % path) 167 | with open(path, 'w') as f: 168 | f.write(contents) 169 | return path 170 | 171 | def setup_module(self, name): 172 | for path in [GIT_DIR + '/%s.git', CACHE_DIR + '/%s']: 173 | self.touch(os.path.join(self.topdir, path % name, '.keep')) 174 | 175 | def assertFileExists(self, module_name, filename, hash, mtime=None): 176 | path = os.path.join(self.topdir, CACHE_DIR, module_name, filename, hash, filename) 177 | self.assertTrue(os.path.exists(path), '%s should exist' % path) 178 | if mtime: 179 | self.assertEqual(os.stat(path).st_mtime, mtime) 180 | 181 | @parameterized.expand(EXISTING_MODULES + OLD_FILE_MODULES) 182 | def test_check_existing_file(self, module, ns_module): 183 | resp = self.upload(module, hash=HASH, filename='hello.txt') 184 | self.assertEqual(resp, 'Available\n') 185 | 186 | @parameterized.expand(EXISTING_MODULES) 187 | def test_check_existing_file_with_bad_hash(self, module, ns_module): 188 | resp = self.upload(module, hash='abc', filename='hello.txt') 189 | self.assertEqual(resp, 'Missing\n') 190 | 191 | @parameterized.expand(EXISTING_MODULES) 192 | def test_check_missing_file(self, module, ns_module): 193 | resp = self.upload(module, hash='abc', filename='foo.txt') 194 | self.assertEqual(resp, 'Missing\n') 195 | 196 | @parameterized.expand(EXISTING_MODULES) 197 | def test_upload_file(self, module, ns_module): 198 | test_file = self.touch('new.txt') 199 | resp = self.upload(module, hash=NEW_HASH, filepath=test_file) 200 | self.assertEqual(resp, 'File new.txt size 7 MD5 %s stored OK\n' % NEW_HASH) 201 | self.assertFileExists(ns_module, 'new.txt', NEW_HASH) 202 | self.assertFileExists(ns_module, 'new.txt', 'md5/' + NEW_HASH) 203 | 204 | @parameterized.expand(EXISTING_MODULES) 205 | def test_upload_file_bad_checksum(self, module, ns_module): 206 | test_file = self.touch('hello.txt') 207 | resp = self.upload(module, hash='ABC', filepath=test_file) 208 | self.assertEqual(resp, 'MD5 check failed. Received %s instead of ABC.\n' % HASH) 209 | 210 | @parameterized.expand(NON_EXISTING_MODULES) 211 | def test_upload_to_non_existing_module(self, module, ns_module): 212 | test_file = self.touch('hello.txt') 213 | resp = self.upload(module, hash=HASH, filepath=test_file) 214 | self.assertEqual(resp, 'Module "%s" does not exist!\n' % ns_module) 215 | 216 | @parameterized.expand(EXISTING_MODULES) 217 | def test_rejects_unknown_hash(self, module, ns_module): 218 | test_file = self.touch('hello.txt') 219 | resp = self.upload(module, hash='deadbeef', hashtype='crc32', filepath=test_file) 220 | self.assertEqual(resp, "Required checksum is not present\n") 221 | 222 | @parameterized.expand(EXISTING_MODULES) 223 | def test_accepts_sha_512_hash(self, module, ns_module): 224 | test_file = self.touch('new.txt') 225 | resp = self.upload(module, hash=NEW_SHA512, hashtype='sha512', filepath=test_file) 226 | self.assertEqual(resp, 'File new.txt size 7 SHA512 %s stored OK\n' % NEW_SHA512) 227 | self.assertFileExists(ns_module, 'new.txt', 'sha512/' + NEW_SHA512) 228 | 229 | @parameterized.expand(EXISTING_MODULES) 230 | def test_bad_sha512_hash(self, module, ns_module): 231 | test_file = self.touch('hello.txt') 232 | resp = self.upload(module, hash='ABC', hashtype='sha512', filepath=test_file) 233 | self.assertEqual(resp, 'SHA512 check failed. Received %s instead of ABC.\n' % SHA512) 234 | 235 | @parameterized.expand(EXISTING_MODULES) 236 | def test_check_existing_sha512_correct(self, module, ns_module): 237 | resp = self.upload(module, hash=SHA512, hashtype='sha512', filename='hello.txt') 238 | self.assertEqual(resp, 'Available\n') 239 | 240 | @parameterized.expand(EXISTING_MODULES) 241 | def test_check_existing_sha512_mismatch(self, module, ns_module): 242 | resp = self.upload(module, hash='abc', hashtype='sha512', filename='hello.txt') 243 | self.assertEqual(resp, 'Missing\n') 244 | 245 | @parameterized.expand(EXISTING_MODULES) 246 | def test_upload_mtime(self, module, ns_module): 247 | test_file = self.touch('new.txt') 248 | resp = self.upload(module, hash=NEW_HASH, filepath=test_file, mtime="1234") 249 | self.assertFileExists(ns_module, 'new.txt', NEW_HASH, mtime=1234) 250 | 251 | @parameterized.expand(EXISTING_MODULES) 252 | def test_upload_invalid_mtime(self, module, ns_module): 253 | test_file = self.touch('new.txt') 254 | resp = self.upload(module, hash=NEW_HASH, filepath=test_file, mtime="abc") 255 | self.assertEqual(resp, 'Invalid value sent for mtime "abc". Aborting.\n') 256 | 257 | 258 | def _copy_tweak(source_file, dest_file, topdir): 259 | """Copy the script from source_file to dest_file, and tweak constants to 260 | point to topdir. 261 | """ 262 | regex = re.compile(r'''^(GITREPO|CACHE_DIR)\s*=\s*['"]([^'"]+)['"]$''') 263 | with open(source_file) as source: 264 | with open(dest_file, 'w') as dest: 265 | for line in source: 266 | 267 | m = regex.match(line) 268 | if m: 269 | line = "%s = '%s%s'\n" % (m.group(1), topdir, m.group(2)) 270 | dest.write(line) 271 | 272 | def _dump_config_file(topdir): 273 | config = ConfigParser() 274 | config["dist-git"] = { 275 | "git_author_name": "Fedora Release Engineering", 276 | "git_author_email": "rel-eng@lists.fedoraproject.org", 277 | "cache_dir": CACHE_DIR, 278 | "lookaside_dir": CACHE_DIR, 279 | "gitroot_dir": GIT_DIR, 280 | "gitolite": True, 281 | "grok": True, 282 | "default_namespace": "rpms", 283 | } 284 | 285 | config["upload"] = { 286 | "fedmsgs": False, 287 | "old_paths": True, 288 | "nomd5": False, 289 | "disable_group_check": True, 290 | } 291 | 292 | tmp = os.path.join(topdir, "dist-git-test.conf") 293 | with open(tmp, "w") as configfile: 294 | config.write(configfile) 295 | return tmp 296 | --------------------------------------------------------------------------------