├── .gitignore ├── LICENSE ├── README.md ├── obsgit ├── __init__.py └── obsgit.py ├── requirements.txt ├── setup.py └── tests └── test_obsgit.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # obsgit 2 | 3 | A simple bridge between Open Build Server and git. 4 | 5 | These tools can be used to export a project stored in OBS into a local 6 | git repository, and later imported from git to the same (or different) 7 | OBS server. 8 | 9 | ## Installation 10 | Install `obsgit` using Python Pip: 11 | 12 | ```bash 13 | pip install git+https://github.com/aplanas/obsgit.git 14 | ``` 15 | 16 | After installed, `obsgit` will be registered as a command-line tool. 17 | 18 | ## Configuration 19 | 20 | `obsgit` requires a configuration file to adjust the parameters of the 21 | different OBS services, the remote storage for big files, and some 22 | configuration of the layout of the local git repository. 23 | 24 | You can generate a default configuration file with: 25 | 26 | ```bash 27 | obsgit create-config 28 | ``` 29 | 30 | This command accepts parameters to adjust the configuration, but you 31 | can also edit the generated file to set the passwords and the 32 | different URL for the APIs: 33 | 34 | ```ini 35 | [export] 36 | # API URL for the build system where we will export from (to git) 37 | url = https://api.opensuse.org 38 | # Login credentials 39 | username = user 40 | password = password 41 | # Only if OBS is configured with SSH authentication (like in IBS) 42 | ssh-key = id_rsa 43 | # What to do when obsgit read a linked package: 44 | # - always: always expand the _link, downloading the expanded source 45 | # - never: never expand, download only the _link file. If link is 46 | # pointing to a different project, generate an error for this package 47 | # - auto: expand the link only if point to a different project 48 | link = never 49 | 50 | [import] 51 | # API URL for the build system where we will upload the project (from git) 52 | url = https://api.opensuse.org 53 | username = user 54 | password = password 55 | 56 | [git] 57 | # Directory name used to store all the packages. If missing, the packages 58 | # will be stored under the git repository 59 | prefix = packages 60 | 61 | [storage] 62 | # Type of repository where to store large binary files 63 | # - lfs: use git-lfs protocol (require git-lfs binary) 64 | # - obs: use OBS / IBS to store the large files 65 | type = lfs 66 | # (obs) API URL for the build system to store files 67 | # url = https://api.opensuse.org 68 | # username = user 69 | # password = password 70 | # (obs) Repository and package where to store the files 71 | # storage = home:user:storage/files 72 | ``` 73 | 74 | ## Export from OBS to git 75 | 76 | The `export` sub-command can be used to read all the metadata of an 77 | OBS project, the list of packages and the content, and download them 78 | in a local git repository. This information is organized with goals in 79 | mind. One is to collect all the information required to re-publish the 80 | project and packages in a different OBS instance, and the other one is 81 | to delegate into git the management of the package assets (changelog, 82 | spec files, patches, etc). 83 | 84 | To export a full project: 85 | 86 | ```bash 87 | obsgit export openSUSE:Factory ~/Project/factory-git 88 | ``` 89 | 90 | If required, this command will initialize the local git repository 91 | given as a second parameter, and using the credentials from the 92 | configuration file, download all the metadata and packages from the 93 | project. 94 | 95 | We can also export a single package: 96 | 97 | ```bash 98 | obsgit export --package gcc openSUSE:Factory ~/Project/factory-git 99 | ``` 100 | 101 | Both commands will read the metadata that OBS stores for the packages 102 | and or the project, and will replace the one that is stored in the 103 | local git repository. Sometimes we do not want to replace the local 104 | metadata, and for that, we can use the `--skip-all-project-meta` and 105 | `--skip-all-package-meta` parameters, or `--skip-project-meta` if we 106 | want only to skip the update for the `_meta` metadata. For example: 107 | 108 | ```bash 109 | obsgit export --skip-project-meta openSUSE:Factory ~/Project/factory-git 110 | ``` 111 | 112 | If we are using the `lfs` extension of git, the export will create a 113 | `.gitattributes` file that references all the detected binary 114 | files. You can use the `git lfs` commands to add or remove tracked 115 | files, add them to the index and do the commit. 116 | 117 | When the storage is configured to use `obs`, the binary files are 118 | uploaded into the storage server and tracked in the 119 | `/.obs/files` metadata file. 120 | 121 | ## Import from git to OBS 122 | 123 | We can re-create the original project that we exported from OBS to git 124 | into a different build service. To do that we can use the `import` 125 | sub-command: 126 | 127 | ```bash 128 | obsgit import ~/Project/factory-git home:user:import 129 | ``` 130 | 131 | In the same way, we can use the `--package` parameter to restrict the 132 | import to a single package, and the different skip metadata 133 | parameters. 134 | 135 | During the `export` stage, the tool collected the metadata information 136 | of the project and for each package. This metadata will contain 137 | information about users that do not exist in the new imported OBS, and 138 | also will contain references to the name of the exported project. 139 | 140 | The `import` stage will try to re-allocate the project into the new 141 | OBS location, editing on the fly the metadata. This edit is basically 142 | a project name replacement: every time the old project name is found 143 | gets replaced with the new project name. If you edit the project name 144 | in the metadata, please, consider updating all the metadata 145 | information for the rest of the files, as `obsgit` will not be able to 146 | re-allocate the project anymore. 147 | 148 | ## Updating the release version of packages 149 | 150 | We can export into an external file OBS revision of the packages 151 | inside a project, and use this number to adjust the revision in the 152 | spec file transparently. 153 | 154 | To fetch only the revision number of the packages, without exporting 155 | anything else: 156 | 157 | ```bash 158 | obsgit export --only-export-revisions revisions.csv openSUSE:Factory ~/Project/factory-git 159 | ``` 160 | 161 | This will create a local file `revisions.csv` that will contain the 162 | name of the package and the last revision (number of commits) 163 | registered by OBS. 164 | 165 | We can use this file to transparently replace the `Release: 0` present 166 | in some spec files during the import. 167 | 168 | ```bash 169 | obsgit import --adjust-release revisions.csv ~/Project/factory-git home:user:import 170 | ``` 171 | 172 | Optionally, you can provide a different CSV file generated, maybe, 173 | analyzing a repository using a different tool. 174 | -------------------------------------------------------------------------------- /obsgit/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openSUSE/obsgit/16b1c55591b38b5ea990911f6ccfa1910ef55232/obsgit/__init__.py -------------------------------------------------------------------------------- /obsgit/obsgit.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import asyncio 3 | import collections 4 | import configparser 5 | import csv 6 | import datetime 7 | import fnmatch 8 | import functools 9 | import getpass 10 | import hashlib 11 | import http 12 | import itertools 13 | import logging 14 | import pathlib 15 | import re 16 | import shutil 17 | import stat 18 | import subprocess 19 | import sys 20 | import time 21 | import xml.etree.ElementTree as ET 22 | 23 | import aiohttp 24 | import chardet 25 | import pygit2 26 | 27 | LOG = logging.getLogger(__name__) 28 | 29 | 30 | def retry(func): 31 | async def wrapper(*args, **kwargs): 32 | to_exception = None 33 | retry = 0 34 | while retry < 5: 35 | try: 36 | return await func(*args, **kwargs) 37 | except asyncio.TimeoutError as e: 38 | to_exception = e 39 | retry += 1 40 | LOG.warning(f"TimeoutError: retry #{retry}") 41 | await asyncio.sleep(0.5) 42 | except http.client.HTTPException as e: 43 | to_exception = e 44 | # Try only one more time 45 | retry = 6 46 | LOG.error("HTTPException: retry one more time") 47 | await asyncio.sleep(0.5) 48 | raise to_exception 49 | 50 | return wrapper 51 | 52 | 53 | # Class based on BasicAuth from aiohttp 54 | class SSHAuth(aiohttp.BasicAuth): 55 | """Http SSH authentication helper.""" 56 | 57 | def __new__(cls, login, password="", ssh_key="", encoding="latin1"): 58 | if login is None: 59 | raise ValueError("None is not allowed as login value") 60 | 61 | if password is None: 62 | raise ValueError("None is not allowed as password value") 63 | 64 | if ssh_key is None: 65 | raise ValueError("None is not allowed as ssh_key value") 66 | 67 | if ":" in login: 68 | raise ValueError('A ":" is not allowed in login (RFC 1945#section-11.1)') 69 | 70 | auth = super().__new__(cls, login, password, encoding) 71 | auth.ssh_key = ssh_key 72 | auth.authorization = "" 73 | auth.already_auth = False 74 | 75 | return auth 76 | 77 | def encode(self): 78 | """Encode credentials.""" 79 | if self.authorization and not self.already_auth: 80 | self.already_auth = True 81 | return self.authorization 82 | 83 | def assert_signature_header(self, headers): 84 | header = [h for h in headers.getall("WWW-Authenticate") if "Signature" in h] 85 | if not header: 86 | raise Exception("Signature authentication not supported in the server") 87 | header = header[0] 88 | 89 | if "Use your developer account" not in header: 90 | raise Exception("Signature realm not expected") 91 | if "(created)" not in header: 92 | raise Exception("Signature header not expected") 93 | 94 | def ssh_sign(self, namespace, data): 95 | cmd = [ 96 | "ssh-keygen", 97 | "-Y", 98 | "sign", 99 | "-f", 100 | self.ssh_key, 101 | "-q", 102 | "-n", 103 | namespace, 104 | "-P", 105 | self.password, 106 | ] 107 | out = subprocess.run( 108 | cmd, 109 | stdout=subprocess.PIPE, 110 | input=data, 111 | stderr=subprocess.STDOUT, 112 | encoding=self.encoding, 113 | ) 114 | lines = out.stdout.splitlines() 115 | 116 | # The signature has a header and a footer. Extract them and 117 | # validate the output. 118 | header, signature, footer = lines[0], lines[1:-1], lines[-1] 119 | if header != "-----BEGIN SSH SIGNATURE-----": 120 | raise Exception(f"Error signing the data: {out.stdout}") 121 | if footer != "-----END SSH SIGNATURE-----": 122 | raise Exception(f"Error signing the data: {out.stdout}") 123 | 124 | return "".join(signature) 125 | 126 | def set_challenge(self, headers): 127 | # TODO: the specification support different headers, and the 128 | # real / namespace should be extracted from the headers 129 | # 130 | # For more complete implementations, check: 131 | # 132 | # * https://datatracker.ietf.org/doc/draft-ietf-httpbis-message-signatures/ 133 | # * https://github.com/openSUSE/osc/pull/1032 134 | # * https://github.com/crazyscientist/osc-tiny 135 | # 136 | self.assert_signature_header(headers) 137 | created = int(time.time()) 138 | namespace = "Use your developer account" 139 | data = f"(created): {created}" 140 | signature = self.ssh_sign(namespace, data) 141 | 142 | self.authorization = ( 143 | f'Signature KeyId="{self.login}",algorithm="ssh",signature={signature},' 144 | f'headers="(created)",created={created}' 145 | ) 146 | 147 | 148 | class ClientRequest(aiohttp.ClientRequest): 149 | def __init__(self, *args, **kwargs): 150 | super().__init__(*args, **kwargs) 151 | 152 | def update_auth(self, auth): 153 | if auth and isinstance(auth, SSHAuth): 154 | # We do not want to include the authorization header in 155 | # each request, as it would overload the server. In fact, 156 | # we already have a cookie in the session that will 157 | # validate subsequent requests. 158 | if auth.already_auth: 159 | return 160 | super().update_auth(auth) 161 | 162 | 163 | class AsyncOBS: 164 | """Minimal asynchronous interface for OBS""" 165 | 166 | def __init__( 167 | self, url, username, password, ssh_key=None, link="auto", verify_ssl=True 168 | ): 169 | self.url = url 170 | self.username = username 171 | self.link = link 172 | 173 | # The key can come from a parameter or from the config file. 174 | # This last one only accept "strings", as is a ConfigParser 175 | if ssh_key: 176 | ssh_key = pathlib.Path(ssh_key) 177 | 178 | if ssh_key and not ssh_key.exists(): 179 | # Give a second chance in user directory 180 | new_ssh_key = pathlib.Path.home() / ".ssh" / ssh_key 181 | if not new_ssh_key.exists(): 182 | raise Exception(f"SSH key not found: {ssh_key}") 183 | ssh_key = new_ssh_key 184 | 185 | conn = aiohttp.TCPConnector(limit=5, limit_per_host=5, verify_ssl=verify_ssl) 186 | if ssh_key: 187 | auth = SSHAuth(username, password, ssh_key) 188 | else: 189 | auth = aiohttp.BasicAuth(username, password) 190 | self.client = aiohttp.ClientSession( 191 | connector=conn, request_class=ClientRequest, auth=auth 192 | ) 193 | 194 | async def close(self): 195 | """Close the client session""" 196 | 197 | # This method must be called at the end of the object 198 | # livecycle. Check aiohttp documentation for details 199 | if self.client: 200 | await self.client.close() 201 | self.client = None 202 | 203 | @retry 204 | async def create(self, project, package=None, disabled=False): 205 | """Create a project and / or package""" 206 | if await self.authorized(project) and not await self.exists(project): 207 | # TODO: generate the XML via ElementTree and ET.dump(root) 208 | if not disabled: 209 | data = ( 210 | f'<description/>' 211 | f'<person userid="{self.username}" role="maintainer"/>' 212 | "</project>" 213 | ) 214 | else: 215 | data = ( 216 | f'<project name="{project}"><title/><description/>' 217 | f'<person userid="{self.username}" role="maintainer"/>' 218 | "<build><disable/></build><publish><disable/></publish>" 219 | "<useforbuild><disable/></useforbuild></project>" 220 | ) 221 | LOG.debug(f"Creating remote project {project} [disabled: {disabled}]") 222 | await self.client.put(f"{self.url}/source/{project}/_meta", data=data) 223 | 224 | if ( 225 | package 226 | and await self.authorized(project, package) 227 | and not await self.exists(project, package) 228 | ): 229 | if not disabled: 230 | data = ( 231 | f'<package name="{package}" project="{project}"><title/>' 232 | "<description/></package>" 233 | ) 234 | else: 235 | data = ( 236 | f'<package name="{package}" project="{project}"><title/>' 237 | "<description/><build><disable/></build><publish><disable/>" 238 | "</publish><useforbuild><disable/></useforbuild></package>" 239 | ) 240 | LOG.debug( 241 | f"Creating remote package {project}/{package} [disabled: {disabled}]" 242 | ) 243 | await self.client.put( 244 | f"{self.url}/source/{project}/{package}/_meta", data=data 245 | ) 246 | 247 | @retry 248 | async def _download(self, url_path, filename_path, **params): 249 | LOG.debug(f"Start download {url_path} to {filename_path}") 250 | async with self.client.get(f"{self.url}/{url_path}", params=params) as resp: 251 | with filename_path.open("wb") as f: 252 | while True: 253 | chunk = await resp.content.read(1024 * 4) 254 | if not chunk: 255 | break 256 | f.write(chunk) 257 | LOG.debug(f"End download {url_path} to {filename_path}") 258 | 259 | async def download(self, project, *path, filename_path, **params): 260 | """Download a file from a project or package""" 261 | url_path = "/".join(("source", project, *path)) 262 | await self._download(url_path, filename_path, **params) 263 | 264 | @retry 265 | async def _upload( 266 | self, url_path, filename_path=None, data=None, headers=None, **params 267 | ): 268 | if filename_path: 269 | LOG.debug(f"Start upload {filename_path} to {url_path}") 270 | with filename_path.open("rb") as f: 271 | resp = await self.client.put( 272 | f"{self.url}/{url_path}", data=f, headers=headers, params=params 273 | ) 274 | LOG.debug(f"End upload {filename_path} to {url_path}") 275 | elif data is not None: 276 | LOG.debug(f"Start upload to {url_path}") 277 | resp = await self.client.put( 278 | f"{self.url}/{url_path}", data=data, headers=headers, params=params 279 | ) 280 | LOG.debug(f"End upload to {url_path}") 281 | else: 282 | resp = None 283 | LOG.warning("Filename nor data provided. Nothing to upload") 284 | 285 | if resp and resp.status >= 400: 286 | raise http.client.HTTPException(f"PUT {resp.status} on {url_path}") 287 | 288 | async def upload( 289 | self, project, *path, filename_path=None, data=None, headers=None, **params 290 | ): 291 | """Upload a file to a project or package""" 292 | url_path = "/".join(("source", project, *path)) 293 | await self._upload( 294 | url_path, filename_path=filename_path, data=data, headers=headers, **params 295 | ) 296 | 297 | @retry 298 | async def _delete(self, url_path, **params): 299 | LOG.debug(f"Delete {url_path}") 300 | await self.client.delete(f"{self.url}/{url_path}", params=params) 301 | 302 | async def delete(self, project, *path, **params): 303 | """Delete a file, project or package""" 304 | url_path = "/".join(("source", project, *path)) 305 | await self._delete(url_path, **params) 306 | 307 | @retry 308 | async def _command(self, url_path, cmd, filename_path=None, data=None, **params): 309 | params["cmd"] = cmd 310 | if filename_path: 311 | LOG.debug(f"Start command {cmd} {filename_path} to {url_path}") 312 | with filename_path.open("rb") as f: 313 | await self.client.post(f"{self.url}/{url_path}", data=f, params=params) 314 | LOG.debug(f"End command {cmd} {filename_path} to {url_path}") 315 | elif data: 316 | LOG.debug(f"Start command {cmd} to {url_path}") 317 | await self.client.post(f"{self.url}/{url_path}", data=data, params=params) 318 | LOG.debug(f"End command {cmd} to {url_path}") 319 | 320 | async def command( 321 | self, project, *path, cmd, filename_path=None, data=None, **params 322 | ): 323 | """Send a command to a project or package""" 324 | url_path = "/".join(("source", project, *path)) 325 | await self._command( 326 | url_path, cmd, filename_path=filename_path, data=data, **params 327 | ) 328 | 329 | @retry 330 | async def _transfer(self, url_path, to_url_path, to_obs=None, **params): 331 | to_obs = to_obs if to_obs else self 332 | LOG.debug(f"Start transfer from {url_path} to {to_url_path}") 333 | resp = await self.client.get(f"{self.url}/{url_path}") 334 | to_url = to_obs.url if to_obs else self.url 335 | await to_obs.client.put( 336 | f"{to_url}/{to_url_path}", data=resp.content, params=params 337 | ) 338 | LOG.debug(f"End transfer from {url_path} to {to_url_path}") 339 | 340 | async def transfer( 341 | self, 342 | project, 343 | package, 344 | filename, 345 | to_project, 346 | to_package=None, 347 | to_filename=None, 348 | to_obs=None, 349 | **params, 350 | ): 351 | """Copy a file between (two) OBS instances""" 352 | to_package = to_package if to_package else package 353 | to_filename = to_filename if to_filename else filename 354 | await self._transfer( 355 | f"source/{project}/{package}/{filename}", 356 | f"source/{to_project}/{to_package}/{to_filename}", 357 | to_obs, 358 | **params, 359 | ) 360 | 361 | @retry 362 | async def _xml(self, url_path, **params): 363 | LOG.debug(f"Fetching XML {url_path}") 364 | try: 365 | async with self.client.get(f"{self.url}/{url_path}", params=params) as resp: 366 | return ET.fromstring(await resp.read()) 367 | except Exception: 368 | return ET.fromstring('<directory rev="latest"/>') 369 | 370 | async def packages(self, project): 371 | """List of packages inside an OBS project""" 372 | root = await self._xml(f"source/{project}") 373 | return [entry.get("name") for entry in root.findall(".//entry")] 374 | 375 | async def files_md5_revision(self, project, package): 376 | """List of (filename, md5) for a package, and the active revision""" 377 | root = await self._xml(f"/source/{project}/{package}", rev="latest") 378 | 379 | revision = root.get("rev") 380 | 381 | if root.find(".//entry[@name='_link']") is not None: 382 | project_link = ( 383 | await self._xml(f"/source/{project}/{package}/_link", rev="latest") 384 | ).get("project") 385 | 386 | if project_link and project_link != project and self.link == "never": 387 | LOG.error( 388 | f"ERROR: Link {project}/{package} pointing outside ({project_link})" 389 | ) 390 | return [], None 391 | 392 | if ( 393 | project_link and project_link != project and self.link == "auto" 394 | ) or self.link == "always": 395 | revision = root.find(".//linkinfo").get("xsrcmd5") 396 | root = await self._xml(f"/source/{project}/{package}", rev=revision) 397 | 398 | files_md5 = [ 399 | (entry.get("name"), entry.get("md5")) for entry in root.findall(".//entry") 400 | ] 401 | 402 | return files_md5, revision 403 | 404 | async def revision(self, project, package): 405 | """Return the active revision of a package""" 406 | root = await self._xml(f"/source/{project}/{package}", rev="latest") 407 | 408 | revision = root.get("rev") 409 | 410 | if root.find(".//entry[@name='_link']") is not None: 411 | project_link = ( 412 | await self._xml(f"/source/{project}/{package}/_link", rev="latest") 413 | ).get("project") 414 | 415 | if project_link and project_link != project and self.link == "never": 416 | LOG.error( 417 | f"ERROR: Link {project}/{package} pointing outside ({project_link})" 418 | ) 419 | return None 420 | 421 | project_link = project_link if project_link else project 422 | package_link = root.find(".//linkinfo").get("package") 423 | root = await self._xml( 424 | f"/source/{project_link}/{package_link}", rev="latest" 425 | ) 426 | revision = root.get("rev") 427 | 428 | return revision 429 | 430 | @retry 431 | async def exists(self, project, package=None): 432 | """Check if a project or package exists in OBS""" 433 | url = ( 434 | f"{self.url}/source/{project}/{package}" 435 | if package 436 | else f"{self.url}/source/{project}" 437 | ) 438 | async with self.client.head(url) as resp: 439 | return resp.status != 404 440 | 441 | @retry 442 | async def authorized(self, project, package=None): 443 | """Check if the user is authorized to access the project or package""" 444 | url = ( 445 | f"{self.url}/source/{project}/{package}" 446 | if package 447 | else f"{self.url}/source/{project}" 448 | ) 449 | async with self.client.head(url) as resp: 450 | if isinstance(self.client._default_auth, SSHAuth): 451 | if not self.client._default_auth.authorization: 452 | self.client._default_auth.set_challenge(resp.headers) 453 | return await self.authorized(project, package) 454 | return resp.status != 401 455 | 456 | 457 | class Git: 458 | """Local git repository""" 459 | 460 | def __init__(self, path, prefix=None): 461 | self.path = pathlib.Path(path) 462 | self.prefix = self.path / prefix if prefix else self.path 463 | self.first_entry = {} 464 | 465 | # TODO: Extend it to packages and files 466 | def exists(self): 467 | """Check if the path is a valid git repository""" 468 | return (self.path / ".git").exists() 469 | 470 | def create(self): 471 | """Create a local git repository""" 472 | self.prefix.mkdir(parents=True, exist_ok=True) 473 | # Convert the path to string, to avoid some limitations in 474 | # older pygit2 475 | pygit2.init_repository(str(self.path)) 476 | 477 | async def delete(self, package, filename=None): 478 | """Delete a package or a file from a git repository""" 479 | loop = asyncio.get_running_loop() 480 | if filename: 481 | await loop.run_in_executor(None, (self.prefix / package / filename).unlink) 482 | else: 483 | await loop.run_in_executor(None, shutil.rmtree, self.prefix / package) 484 | 485 | def packages(self): 486 | """List of packages in the git repository""" 487 | return [ 488 | package.parts[-1] 489 | for package in self.prefix.iterdir() 490 | if package.is_dir() and package.parts[-1] not in (".git", ".obs") 491 | ] 492 | 493 | def _md5(self, package, filename): 494 | md5 = hashlib.md5() 495 | with (self.prefix / package / filename).open("rb") as f: 496 | while True: 497 | chunk = f.read(1024 * 4) 498 | if not chunk: 499 | break 500 | md5.update(chunk) 501 | return md5.hexdigest() 502 | 503 | async def files_md5(self, package): 504 | """List of (filename, md5) for a package""" 505 | # TODO: For Python >= 3.7 use get_running_loop() 506 | loop = asyncio.get_event_loop() 507 | files = [ 508 | file_.parts[-1] 509 | for file_ in (self.prefix / package).iterdir() 510 | if file_.is_file() 511 | ] 512 | md5s = await asyncio.gather( 513 | *( 514 | loop.run_in_executor(None, self._md5, package, filename) 515 | for filename in files 516 | ) 517 | ) 518 | return zip(files, md5s) 519 | 520 | def head_hash(self): 521 | return pygit2.Repository(str(self.path)).head.target 522 | 523 | def _patches(self): 524 | repo = pygit2.Repository(str(self.path)) 525 | last = repo[repo.head.target] 526 | for commit in repo.walk( 527 | last.id, pygit2.GIT_SORT_TOPOLOGICAL | pygit2.GIT_SORT_TIME 528 | ): 529 | if len(commit.parents) == 1: 530 | for patch in commit.tree.diff_to_tree(commit.parents[0].tree): 531 | yield commit, patch 532 | elif len(commit.parents) == 0: 533 | for patch in commit.tree.diff_to_tree(): 534 | yield commit, patch 535 | 536 | def analyze_history(self): 537 | packages_path = { 538 | (self.prefix / package).relative_to(self.path) 539 | for package in self.packages() 540 | } 541 | 542 | for commit, patch in self._patches(): 543 | packages = packages_path & set( 544 | pathlib.Path(patch.delta.new_file.path).parents 545 | ) 546 | assert len(packages) <= 1 547 | if packages: 548 | package = packages.pop() 549 | self.first_entry.setdefault( 550 | package, 551 | ( 552 | commit.oid, 553 | commit.author.name, 554 | commit.author.email, 555 | datetime.datetime.utcfromtimestamp(commit.commit_time), 556 | ), 557 | ) 558 | 559 | def last_revision_to(self, package): 560 | package_path = (self.prefix / package).relative_to(self.path) 561 | return self.first_entry.get(package_path) 562 | 563 | 564 | class StorageOBS: 565 | """File storage in OBS""" 566 | 567 | async def __new__(cls, *args, **kwargs): 568 | instance = super().__new__(cls) 569 | await instance.__init__(*args, **kwargs) 570 | return instance 571 | 572 | async def __init__(self, obs, project, package, git): 573 | self.obs = obs 574 | self.project = project 575 | self.package = package 576 | self.git = git 577 | 578 | self.index = set() 579 | self.sync = True 580 | 581 | await self._update_index() 582 | 583 | async def _update_index(self): 584 | # TODO: we do not clean the index, we only add elements 585 | files_md5, _ = await self.obs.files_md5_revision(self.project, self.package) 586 | for filename, md5 in files_md5: 587 | assert filename == md5, f"Storage {self.project}/{self.package} not valid" 588 | self.index.add(filename) 589 | 590 | async def transfer(self, md5, project, package, filename, obs, **params): 591 | """Copy a file to the file storage from a remote OBS""" 592 | assert ( 593 | md5 in self.index 594 | ), f"File {package}/{filename} ({md5}) missing from storage" 595 | # TODO: replace "transfer" with copy_to and copy_from. 596 | # TODO: when both OBS services are the same, use the copy pack 597 | # / commit trick from 598 | # https://github.com/openSUSE/open-build-service/issues/9615 599 | print(f"(StorageOBS) transfering {project}/{package}/{filename}") 600 | await self.obs.transfer( 601 | self.project, self.package, md5, project, package, filename, obs, **params 602 | ) 603 | print(f"(StorageOBS) transferred {project}/{package}/{filename}") 604 | 605 | async def _store(self, filename_path, md5): 606 | """Store a file with md5 into the storage""" 607 | self.index.add(md5) 608 | self.sync = False 609 | 610 | print(f"(StorageOBS) storing {filename_path}") 611 | await self.obs.upload( 612 | self.project, 613 | self.package, 614 | md5, 615 | filename_path=filename_path, 616 | rev="repository", 617 | ) 618 | print(f"(StorageOBS) stored {filename_path}") 619 | 620 | async def store_files(self, package, files_md5): 621 | package_path = self.git.prefix / package 622 | files_md5_exists = [ 623 | (filename, md5) 624 | for filename, md5 in files_md5 625 | if (package_path / filename).exists() 626 | ] 627 | 628 | await asyncio.gather( 629 | *( 630 | self._store(package_path / filename, md5) 631 | for filename, md5 in files_md5_exists 632 | ) 633 | ) 634 | 635 | await asyncio.gather( 636 | *(self.git.delete(package, filename) for filename, _ in files_md5_exists) 637 | ) 638 | 639 | with (package_path / ".obs" / "files").open("w") as f: 640 | f.writelines( 641 | f"{filename}\t\t{md5}\n" for filename, md5 in sorted(files_md5) 642 | ) 643 | 644 | async def fetch(self, md5, filename_path): 645 | """Download a file from the storage under a different filename""" 646 | self.obs.download(self.project, self.package, md5, filename_path=filename_path) 647 | 648 | async def commit(self): 649 | # If the index is still valid, we do not commit a change 650 | if self.sync: 651 | return 652 | 653 | directory = ET.Element("directory") 654 | for md5 in self.index: 655 | entry = ET.SubElement(directory, "entry") 656 | entry.attrib["name"] = md5 657 | entry.attrib["md5"] = md5 658 | commit_date = datetime.datetime.now().strftime("%d/%m/%Y %H:%M:%S") 659 | await self.obs.command( 660 | self.project, 661 | self.package, 662 | cmd="commitfilelist", 663 | data=ET.tostring(directory), 664 | user=self.obs.username, 665 | comment=f"Storage syncronization {commit_date}", 666 | ) 667 | self.sync = True 668 | 669 | 670 | class StorageLFS: 671 | """File storage in git LFS""" 672 | 673 | def __init__(self, git): 674 | self.git = git 675 | # When using the OBS storage we can avoid some downloads, but 676 | # is not the case for LFS. In this model the index will be 677 | # empty always. 678 | self.index = set() 679 | self.tracked = set() 680 | 681 | self._update_tracked() 682 | 683 | def _update_tracked(self): 684 | out = subprocess.run( 685 | ["git", "lfs", "track"], 686 | cwd=self.git.path, 687 | stdout=subprocess.PIPE, 688 | stderr=subprocess.STDOUT, 689 | encoding="utf-8", 690 | ) 691 | for line in out.stdout.splitlines(): 692 | if line.startswith(" " * 4): 693 | self.tracked.add(line.split()[0]) 694 | 695 | async def is_installed(self): 696 | out = subprocess.run( 697 | ["git", "lfs", "install"], 698 | cwd=self.git.path, 699 | stdout=subprocess.PIPE, 700 | stderr=subprocess.STDOUT, 701 | ) 702 | is_installed = out.returncode == 0 703 | 704 | # Track the default extensions already, we can later include 705 | # specific files 706 | if is_installed: 707 | for binary in Exporter.BINARY | Exporter.NON_BINARY_EXCEPTIONS: 708 | await self._store(pathlib.Path(f"*{binary}")) 709 | 710 | return is_installed 711 | 712 | def overlaps(self): 713 | return [ 714 | (a, b) 715 | for a, b in itertools.combinations(self.tracked, 2) 716 | if fnmatch.fnmatch(a, b) 717 | ] 718 | 719 | def transfer(self, md5, project, package, filename, obs): 720 | pass 721 | 722 | def _tracked(self, filename): 723 | return any(fnmatch.fnmatch(filename, track) for track in self.tracked) 724 | 725 | async def _store(self, filename_path): 726 | # When registering general patterms, like "*.gz" we do not 727 | # have a path relative to the git repository 728 | try: 729 | filename_path = filename_path.relative_to(self.git.path) 730 | except ValueError: 731 | pass 732 | 733 | # TODO: we can edit `.gitattributes` manually 734 | subprocess.run( 735 | ["git", "lfs", "track", filename_path], 736 | cwd=self.git.path, 737 | stdout=subprocess.PIPE, 738 | stderr=subprocess.STDOUT, 739 | ) 740 | self.tracked.add(str(filename_path)) 741 | await self.commit() 742 | 743 | async def store_files(self, package, files_md5): 744 | package_path = self.git.prefix / package 745 | for filename, _ in files_md5: 746 | if not self._tracked(filename): 747 | await self._store(package_path / filename) 748 | 749 | async def fetch(self): 750 | pass 751 | 752 | async def delete(self, filename_path): 753 | subprocess.run( 754 | ["git", "lfs", "untrack", filename_path], 755 | cwd=self.git.path, 756 | stdout=subprocess.PIPE, 757 | stderr=subprocess.STDOUT, 758 | ) 759 | 760 | async def commit(self): 761 | subprocess.run( 762 | ["git", "add", ".gitattributes"], 763 | cwd=self.git.path, 764 | stdout=subprocess.PIPE, 765 | stderr=subprocess.STDOUT, 766 | ) 767 | 768 | 769 | class Exporter: 770 | """Export projects and packages from OBS to git""" 771 | 772 | BINARY = { 773 | ".xz", 774 | ".gz", 775 | ".bz2", 776 | ".zip", 777 | ".gem", 778 | ".tgz", 779 | ".png", 780 | ".pdf", 781 | ".jar", 782 | ".oxt", 783 | ".whl", 784 | ".rpm", 785 | } 786 | NON_BINARY_EXCEPTIONS = {".obscpio"} 787 | NON_BINARY = { 788 | ".changes", 789 | ".spec", 790 | ".patch", 791 | ".diff", 792 | ".conf", 793 | ".yml", 794 | ".keyring", 795 | ".sig", 796 | ".sh", 797 | ".dif", 798 | ".txt", 799 | ".service", 800 | ".asc", 801 | ".cabal", 802 | ".desktop", 803 | ".xml", 804 | ".pom", 805 | ".SUSE", 806 | ".in", 807 | ".obsinfo", 808 | ".1", 809 | ".init", 810 | ".kiwi", 811 | ".rpmlintrc", 812 | ".rules", 813 | ".py", 814 | ".sysconfig", 815 | ".logrotate", 816 | ".pl", 817 | ".dsc", 818 | ".c", 819 | ".install", 820 | ".8", 821 | ".md", 822 | ".html", 823 | ".script", 824 | ".xml", 825 | ".test", 826 | ".cfg", 827 | ".el", 828 | ".pamd", 829 | ".sign", 830 | ".macros", 831 | } 832 | 833 | def __init__( 834 | self, 835 | obs, 836 | git, 837 | storage, 838 | skip_project_meta, 839 | skip_all_project_meta, 840 | skip_package_meta, 841 | skip_all_package_meta, 842 | ): 843 | self.obs = obs 844 | self.git = git 845 | self.storage = storage 846 | self.skip_project_meta = skip_project_meta 847 | self.skip_all_project_meta = skip_all_project_meta 848 | self.skip_package_meta = skip_package_meta 849 | self.skip_all_package_meta = skip_all_package_meta 850 | 851 | @staticmethod 852 | def is_binary(filename): 853 | """Use some heuristics to detect if a file is binary""" 854 | # Shortcut the detection based on the file extension 855 | suffix = pathlib.Path(filename).suffix 856 | if suffix in Exporter.BINARY or suffix in Exporter.NON_BINARY_EXCEPTIONS: 857 | return True 858 | if suffix in Exporter.NON_BINARY: 859 | return False 860 | 861 | # Small (5Kb) files are considered as text 862 | if filename.stat().st_size < 5 * 1024: 863 | return False 864 | 865 | # Read a chunk of the file and try to determine the encoding, if 866 | # the confidence is low we assume binary 867 | with filename.open("rb") as f: 868 | chunk = f.read(4 * 1024) 869 | try: 870 | chunk.decode("utf-8") 871 | except UnicodeDecodeError: 872 | encoding = chardet.detect(chunk) 873 | else: 874 | return False 875 | return encoding["confidence"] < 0.8 876 | 877 | async def project(self, project): 878 | """Export a project from OBS to git""" 879 | packages_obs = set(await self.obs.packages(project)) 880 | packages_git = set(self.git.packages()) 881 | packages_delete = packages_git - packages_obs 882 | 883 | if not ((self.git.path / ".obs").exists() and self.skip_all_project_meta): 884 | await self.project_metadata(project) 885 | 886 | await asyncio.gather( 887 | *(self.package(project, package) for package in packages_obs), 888 | *(self.git.delete(package) for package in packages_delete), 889 | ) 890 | 891 | await self.storage.commit() 892 | 893 | async def project_metadata(self, project): 894 | """Export the project metadata from OBS to git""" 895 | metadata_path = self.git.path / ".obs" 896 | metadata_path.mkdir(exist_ok=True) 897 | 898 | metadata = [ 899 | "_project", 900 | "_attribute", 901 | "_config", 902 | "_pattern", 903 | ] 904 | if not self.skip_project_meta: 905 | metadata.append("_meta") 906 | 907 | await asyncio.gather( 908 | *( 909 | self.obs.download(project, meta, filename_path=metadata_path / meta) 910 | for meta in metadata 911 | ) 912 | ) 913 | 914 | async def package(self, project, package): 915 | """Export a package from OBS to git""" 916 | package_path = self.git.prefix / package 917 | package_path.mkdir(exist_ok=True) 918 | 919 | print(f"{project}/{package} ...") 920 | 921 | if not ( 922 | (self.git.prefix / package / ".obs").exists() and self.skip_all_package_meta 923 | ): 924 | await self.package_metadata(project, package) 925 | 926 | # We do not know, before downloading, if a file is binary or 927 | # text. The strategy for now is to download all the files 928 | # (except the ones already in the remote storage or in git), 929 | # and upload later the ones that are binary. We need to 930 | # remove those after that 931 | 932 | files_md5_obs, revision = await self.obs.files_md5_revision(project, package) 933 | files_md5_obs = set(files_md5_obs) 934 | files_md5_git = set(await self.git.files_md5(package)) 935 | 936 | # TODO: one optimization is to detect the files that are 937 | # stored in the local "files" cache, that we already know that 938 | # are binary, and do a transfer if the MD5 is different 939 | files_download = { 940 | filename 941 | for filename, md5 in (files_md5_obs - files_md5_git) 942 | if md5 not in self.storage.index 943 | } 944 | 945 | files_obs = {filename for filename, _ in files_md5_obs} 946 | files_git = {filename for filename, _ in files_md5_git} 947 | files_delete = files_git - files_obs 948 | 949 | await asyncio.gather( 950 | *( 951 | self.obs.download( 952 | project, 953 | package, 954 | filename, 955 | filename_path=package_path / filename, 956 | rev=revision, 957 | ) 958 | for filename in files_download 959 | ), 960 | *(self.git.delete(package, filename) for filename in files_delete), 961 | ) 962 | 963 | # TODO: do not over-optimize here, and detect old binary files 964 | # Once we download the full package, we store the new binary files 965 | files_md5_store = [ 966 | (filename, md5) 967 | for filename, md5 in files_md5_obs 968 | if filename in files_download 969 | and Exporter.is_binary(package_path / filename) 970 | ] 971 | files_md5_obs_store = [ 972 | (filename, md5) 973 | for filename, md5 in files_md5_obs 974 | if md5 in self.storage.index 975 | ] 976 | await self.storage.store_files(package, files_md5_store + files_md5_obs_store) 977 | 978 | async def package_metadata(self, project, package): 979 | metadata_path = self.git.prefix / package / ".obs" 980 | metadata_path.mkdir(exist_ok=True) 981 | 982 | metadata = [ 983 | "_attribute", 984 | # "_history", 985 | ] 986 | if not self.skip_package_meta: 987 | metadata.append("_meta") 988 | 989 | await asyncio.gather( 990 | *( 991 | self.obs.download( 992 | project, package, meta, filename_path=metadata_path / meta 993 | ) 994 | for meta in metadata 995 | ) 996 | ) 997 | 998 | async def export_revisions(self, project, revisions_csv): 999 | """Export the packages revision numbers from OBS to git""" 1000 | packages_obs = await self.obs.packages(project) 1001 | 1002 | revisions = await asyncio.gather( 1003 | *(self.obs.revision(project, package) for package in packages_obs) 1004 | ) 1005 | 1006 | with revisions_csv.open("w") as f: 1007 | writer = csv.writer(f) 1008 | writer.writerows(zip(packages_obs, revisions)) 1009 | 1010 | 1011 | class Importer: 1012 | def __init__( 1013 | self, 1014 | obs, 1015 | git, 1016 | storage, 1017 | remove_role_project_meta, 1018 | skip_project_meta, 1019 | skip_all_project_meta, 1020 | remove_role_package_meta, 1021 | skip_package_meta, 1022 | skip_all_package_meta, 1023 | skip_changes_commit_hash, 1024 | ): 1025 | self.obs = obs 1026 | self.git = git 1027 | self.storage = storage 1028 | self.remove_role_project_meta = remove_role_project_meta 1029 | self.skip_project_meta = skip_project_meta 1030 | self.skip_all_project_meta = skip_all_project_meta 1031 | self.remove_role_package_meta = remove_role_package_meta 1032 | self.skip_package_meta = skip_package_meta 1033 | self.skip_all_package_meta = skip_all_package_meta 1034 | self.skip_changes_commit_hash = skip_changes_commit_hash 1035 | 1036 | self._revisions = {} 1037 | 1038 | @functools.lru_cache() 1039 | def project_name(self): 1040 | metadata_path = self.git.path / ".obs" / "_meta" 1041 | return ET.parse(metadata_path).getroot().get("name") 1042 | 1043 | def adjust_metadata( 1044 | self, filename_path, project, project_name=None, remove_role=False 1045 | ): 1046 | # Replace the package name 1047 | project_name = project_name if project_name else self.project_name() 1048 | with filename_path.open() as f: 1049 | meta = f.read().replace(project_name, project) 1050 | 1051 | if remove_role: 1052 | meta = re.sub("<person .*?>", "", meta) 1053 | meta = re.sub("<group .*?>", "", meta) 1054 | return meta 1055 | 1056 | @functools.lru_cache() 1057 | def changes_git_entry(self, package): 1058 | last_revision = self.git.last_revision_to(package) 1059 | 1060 | if not last_revision: 1061 | LOG.error(f"ERROR: {package} not found in git history") 1062 | return "" 1063 | 1064 | commit_hash, author, email, commit_date = self.git.last_revision_to(package) 1065 | entry = "-" * 67 1066 | commit_date = commit_date.strftime("%a %b %d %H:%M:%S UTC %Y") 1067 | entry = f"{entry}\n{commit_date} - {author} <{email}>" 1068 | entry = f"{entry}\n\n- Last git synchronization: {commit_hash}\n\n" 1069 | return entry 1070 | 1071 | def prepend_changes(self, filename_path, package): 1072 | with filename_path.open("rb") as f: 1073 | changes = f.read() 1074 | if not self.skip_changes_commit_hash: 1075 | changes = self.changes_git_entry(package).encode("utf-8") + changes 1076 | return changes 1077 | 1078 | def load_revisions(self, revisions_csv): 1079 | try: 1080 | with revisions_csv.open() as f: 1081 | reader = csv.reader(f) 1082 | self._revisions = dict(reader) 1083 | except Exception: 1084 | LOG.error(f"ERROR: {revisions_csv} not found or not valid") 1085 | 1086 | def adjust_release(self, filename_path, package): 1087 | with filename_path.open("rb") as f: 1088 | spec = f.read() 1089 | revision = self._revisions.get(package) 1090 | if revision: 1091 | spec = re.sub( 1092 | rb"Release\s*:\s*(?:0|<RELEASE>)", 1093 | f"Release: {revision}".encode("utf-8"), 1094 | spec, 1095 | ) 1096 | return spec 1097 | 1098 | async def project(self, project): 1099 | # TODO: What if the project in OBS is more modern? Is there a 1100 | # way to detect it? 1101 | 1102 | # First import the project metadata, as a side effect can 1103 | # create the project 1104 | if not (await self.obs.exists(project) and self.skip_all_project_meta): 1105 | await self.project_metadata(project) 1106 | 1107 | packages_obs = set(await self.obs.packages(project)) 1108 | packages_git = set(self.git.packages()) 1109 | packages_delete = packages_obs - packages_git 1110 | 1111 | # Order the packages, uploading the links the last 1112 | packages_git = sorted( 1113 | packages_git, key=lambda x: (self.git.prefix / x / "_link").exists() 1114 | ) 1115 | 1116 | # To avoid stressing OBS / IBS we group the imports 1117 | # TODO: decide if fully serialize the fetch 1118 | group_size = 4 1119 | packages_git = list(packages_git) 1120 | packages_git_groups = [ 1121 | packages_git[i : i + group_size] 1122 | for i in range(0, len(packages_git), group_size) 1123 | ] 1124 | for packages_git_group in packages_git_groups: 1125 | await asyncio.gather( 1126 | *(self.package(project, package) for package in packages_git_group) 1127 | ) 1128 | 1129 | await asyncio.gather( 1130 | *(self.obs.delete(project, package) for package in packages_delete) 1131 | ) 1132 | 1133 | async def project_metadata(self, project): 1134 | metadata_path = self.git.path / ".obs" 1135 | 1136 | # When creating a new project, we should add first the _meta 1137 | # file, and later the rest 1138 | if not self.skip_project_meta: 1139 | await self.obs.upload( 1140 | project, 1141 | "_meta", 1142 | data=self.adjust_metadata( 1143 | metadata_path / "_meta", 1144 | project, 1145 | remove_role=self.remove_role_project_meta, 1146 | ), 1147 | ) 1148 | 1149 | metadata = [ 1150 | # "_project", 1151 | # "_attribute", 1152 | "_config", 1153 | # "_pattern", 1154 | ] 1155 | 1156 | await asyncio.gather( 1157 | *( 1158 | self.obs.upload( 1159 | project, 1160 | meta, 1161 | data=self.adjust_metadata( 1162 | metadata_path / meta, 1163 | project, 1164 | remove_role=self.remove_role_project_meta, 1165 | ), 1166 | ) 1167 | for meta in metadata 1168 | ) 1169 | ) 1170 | 1171 | async def _git_files_md5(self, package): 1172 | files_md5 = [] 1173 | for filename, md5 in await self.git.files_md5(package): 1174 | filename_path = self.git.prefix / package / filename 1175 | if filename_path.suffix == ".changes": 1176 | md5 = hashlib.md5() 1177 | md5.update(self.prepend_changes(filename_path, package)) 1178 | md5 = md5.hexdigest() 1179 | elif filename_path.suffix == ".spec": 1180 | md5 = hashlib.md5() 1181 | md5.update(self.adjust_release(filename_path, package)) 1182 | md5 = md5.hexdigest() 1183 | files_md5.append((filename, md5)) 1184 | return files_md5 1185 | 1186 | async def package(self, project, package): 1187 | print(f"{project}/{package} ...") 1188 | 1189 | if not (await self.obs.exists(project, package) and self.skip_all_package_meta): 1190 | await self.package_metadata(project, package) 1191 | 1192 | package_path = self.git.prefix / package 1193 | 1194 | files_md5_obs, _ = await self.obs.files_md5_revision(project, package) 1195 | files_md5_obs = set(files_md5_obs) 1196 | files_md5_git = set(await self._git_files_md5(package)) 1197 | 1198 | # TODO: reading the files is part of StorageXXX class 1199 | meta_file = package_path / ".obs" / "files" 1200 | if meta_file.exists(): 1201 | with (meta_file).open() as f: 1202 | files_md5_git_store = {tuple(line.split()) for line in f.readlines()} 1203 | else: 1204 | files_md5_git_store = set() 1205 | 1206 | files_md5_upload = files_md5_git - files_md5_obs 1207 | files_md5_transfer = files_md5_git_store - files_md5_obs 1208 | 1209 | files_obs = {filename for filename, _ in files_md5_obs} 1210 | files_git = {filename for filename, _ in files_md5_git} 1211 | files_git_store = {filename for filename, _ in files_md5_git_store} 1212 | files_delete = files_obs - files_git - files_git_store 1213 | 1214 | await asyncio.gather( 1215 | *( 1216 | self.obs.upload( 1217 | project, 1218 | package, 1219 | filename, 1220 | filename_path=package_path / filename, 1221 | rev="repository", 1222 | ) 1223 | for filename, _ in files_md5_upload 1224 | if not filename.endswith((".changes", ".spec", ".json")) 1225 | ), 1226 | *( 1227 | self.obs.upload( 1228 | project, 1229 | package, 1230 | filename, 1231 | data=self.prepend_changes(package_path / filename, package), 1232 | rev="repository", 1233 | ) 1234 | for filename, _ in files_md5_upload 1235 | if filename.endswith(".changes") 1236 | ), 1237 | *( 1238 | self.obs.upload( 1239 | project, 1240 | package, 1241 | filename, 1242 | data=self.adjust_release(package_path / filename, package), 1243 | rev="repository", 1244 | ) 1245 | for filename, _ in files_md5_upload 1246 | if filename.endswith(".spec") 1247 | ), 1248 | *( 1249 | self.obs.upload( 1250 | project, 1251 | package, 1252 | filename, 1253 | filename_path=package_path / filename, 1254 | headers={"content-type": "text/xml"}, 1255 | rev="repository", 1256 | ) 1257 | for filename, _ in files_md5_upload 1258 | if filename.endswith(".json") 1259 | ), 1260 | *( 1261 | self.storage.transfer( 1262 | md5, project, package, filename, self.obs, rev="repository" 1263 | ) 1264 | for filename, md5 in files_md5_transfer 1265 | ), 1266 | *( 1267 | self.obs.delete(project, package, filename, rev="repository") 1268 | for filename in files_delete 1269 | ), 1270 | ) 1271 | 1272 | if files_md5_upload or files_md5_transfer or files_delete: 1273 | # Create the directory XML to generate a commit 1274 | directory = ET.Element("directory") 1275 | for filename, md5 in files_md5_git | files_md5_git_store: 1276 | entry = ET.SubElement(directory, "entry") 1277 | entry.attrib["name"] = filename 1278 | entry.attrib["md5"] = md5 1279 | 1280 | head_hash = self.git.head_hash() 1281 | 1282 | await self.obs.command( 1283 | project, 1284 | package, 1285 | cmd="commitfilelist", 1286 | data=ET.tostring(directory), 1287 | user=self.obs.username, 1288 | comment=f"Import {head_hash}", 1289 | ) 1290 | 1291 | async def package_metadata(self, project, package): 1292 | metadata_path = self.git.prefix / package / ".obs" 1293 | metadata = ( 1294 | "_meta", 1295 | # "_attribute", 1296 | # "_history", 1297 | ) 1298 | 1299 | # Validate that the metadata can be re-allocated 1300 | project_name = self.project_name() 1301 | package_project_name = ( 1302 | ET.parse(metadata_path / "_meta").getroot().get("project") 1303 | ) 1304 | if project_name != package_project_name: 1305 | LOG.warning(f"Please, edit the metadata for {package}") 1306 | 1307 | await asyncio.gather( 1308 | *( 1309 | self.obs.upload( 1310 | project, 1311 | package, 1312 | meta, 1313 | data=self.adjust_metadata( 1314 | metadata_path / meta, 1315 | project, 1316 | package_project_name, 1317 | remove_role=self.remove_role_package_meta, 1318 | ), 1319 | ) 1320 | for meta in metadata 1321 | ) 1322 | ) 1323 | 1324 | 1325 | def read_config(config_filename): 1326 | """Read or create a configuration file in INI format""" 1327 | if not config_filename: 1328 | print("Configuration file not provided") 1329 | sys.exit(-1) 1330 | 1331 | if not config_filename.exists(): 1332 | print(f"Configuration file {config_filename} not found.") 1333 | print("Use create_config to create a new configuration file") 1334 | sys.exit(-1) 1335 | 1336 | config = configparser.ConfigParser() 1337 | config.read(config_filename) 1338 | 1339 | # Old configuration files do not have the new ssh-key parameter. 1340 | # Provide a default value. 1341 | for section in ("export", "import"): 1342 | if "ssh-key" not in config[section]: 1343 | config[section]["ssh-key"] = "" 1344 | if config["storage"]["type"] == "obs" and "ssh-key" not in config["storage"]: 1345 | config["storage"]["ssh-key"] = "" 1346 | 1347 | return config 1348 | 1349 | 1350 | def create_config(args): 1351 | if not args.config: 1352 | print("Configuration file not provided") 1353 | sys.exit(-1) 1354 | 1355 | config = configparser.ConfigParser() 1356 | 1357 | config["export"] = { 1358 | "url": args.api, 1359 | "username": args.username, 1360 | "password": args.password if args.password else "<password>", 1361 | "ssh-key": args.ssh_key if args.ssh_key else "<ssh-key-path>", 1362 | "link": args.link, 1363 | } 1364 | 1365 | config["import"] = { 1366 | "url": args.api, 1367 | "username": args.username, 1368 | "password": args.password if args.password else "<password>", 1369 | "ssh-key": args.ssh_key if args.ssh_key else "<ssh-key-path>", 1370 | } 1371 | 1372 | if args.storage == "obs": 1373 | config["storage"] = { 1374 | "type": "obs", 1375 | "url": args.api, 1376 | "username": args.username, 1377 | "password": args.password if args.password else "<password>", 1378 | "ssh-key": args.ssh_key if args.ssh_key else "<ssh-key-path>", 1379 | "storage": f"home:{args.username}:storage/files", 1380 | } 1381 | elif args.storage == "lfs": 1382 | config["storage"] = { 1383 | "type": "lfs", 1384 | } 1385 | else: 1386 | print(f"Storage type {args.storage} not valid") 1387 | sys.exit(-1) 1388 | 1389 | config["git"] = {"prefix": args.prefix} 1390 | 1391 | with args.config.open("w") as f: 1392 | config.write(f) 1393 | 1394 | # Only the user can read and write the file 1395 | args.config.chmod(stat.S_IRUSR | stat.S_IWUSR) 1396 | 1397 | print(f"Edit {args.config} to adjust the configuration and passwords") 1398 | 1399 | return config 1400 | 1401 | 1402 | async def export(args, config): 1403 | project = args.project 1404 | repository = pathlib.Path(args.repository).expanduser().absolute().resolve() 1405 | package = args.package 1406 | 1407 | obs = AsyncOBS( 1408 | config["export"]["url"], 1409 | config["export"]["username"], 1410 | config["export"]["password"], 1411 | config["export"]["ssh-key"], 1412 | config["export"]["link"], 1413 | verify_ssl=not args.disable_verify_ssl, 1414 | ) 1415 | 1416 | if not await obs.authorized(project, package): 1417 | print("No authorization to access project or package in build service") 1418 | sys.exit(-1) 1419 | 1420 | if not await obs.exists(project, package): 1421 | print("Project or package not found in build service") 1422 | sys.exit(-1) 1423 | 1424 | git = Git(repository, config["git"]["prefix"]) 1425 | git.create() 1426 | print("Initialized the git repository") 1427 | 1428 | storage_type = config["storage"]["type"] 1429 | if storage_type == "obs": 1430 | storage_obs = AsyncOBS( 1431 | config["storage"]["url"], 1432 | config["storage"]["username"], 1433 | config["storage"]["password"], 1434 | config["storage"]["ssh-key"], 1435 | verify_ssl=not args.disable_verify_ssl, 1436 | ) 1437 | storage_project, storage_package = pathlib.Path( 1438 | config["storage"]["storage"] 1439 | ).parts 1440 | await storage_obs.create(storage_project, storage_package, disabled=True) 1441 | print("Remote storage in OBS initialized") 1442 | 1443 | storage = await StorageOBS(storage_obs, storage_project, storage_package, git) 1444 | elif storage_type == "lfs": 1445 | storage = StorageLFS(git) 1446 | 1447 | if not await storage.is_installed(): 1448 | print("LFS extension not installed") 1449 | await obs.close() 1450 | sys.exit(-1) 1451 | print("Git LFS extension enabled in the repository") 1452 | 1453 | overlaps = storage.overlaps() 1454 | if overlaps: 1455 | print("Multiple LFS tracks are overlaped. Fix them manually.") 1456 | for a, b in overlaps: 1457 | print(f"* {a} - {b}") 1458 | else: 1459 | raise NotImplementedError(f"Storage {storage_type} not implemented") 1460 | 1461 | exporter = Exporter( 1462 | obs, 1463 | git, 1464 | storage, 1465 | args.skip_project_meta, 1466 | args.skip_all_project_meta, 1467 | args.skip_package_meta, 1468 | args.skip_all_package_meta, 1469 | ) 1470 | if args.only_export_revisions: 1471 | await exporter.export_revisions(project, args.only_export_revisions) 1472 | else: 1473 | if package: 1474 | # To have a self consisten unit, maybe we need to export 1475 | # also the project metadata 1476 | if not ((git.path / ".obs").exists() or args.skip_all_project_meta): 1477 | await exporter.project_metadata(project) 1478 | await exporter.package(project, package) 1479 | else: 1480 | await exporter.project(project) 1481 | 1482 | if storage_type == "obs": 1483 | await storage_obs.close() 1484 | await obs.close() 1485 | 1486 | 1487 | async def import_(args, config): 1488 | repository = pathlib.Path(args.repository).expanduser().absolute().resolve() 1489 | project = args.project 1490 | package = args.package 1491 | 1492 | obs = AsyncOBS( 1493 | config["import"]["url"], 1494 | config["import"]["username"], 1495 | config["import"]["password"], 1496 | config["import"]["ssh-key"], 1497 | config["export"]["link"], 1498 | verify_ssl=not args.disable_verify_ssl, 1499 | ) 1500 | 1501 | if not await obs.authorized(project, package): 1502 | print("No authorization to access project or package in build service") 1503 | sys.exit(-1) 1504 | 1505 | git = Git(repository, config["git"]["prefix"]) 1506 | if not git.exists(): 1507 | print("Local git repository is not valid") 1508 | sys.exit(-1) 1509 | git.analyze_history() 1510 | 1511 | storage_type = config["storage"]["type"] 1512 | if storage_type == "obs": 1513 | storage_obs = AsyncOBS( 1514 | config["storage"]["url"], 1515 | config["storage"]["username"], 1516 | config["storage"]["password"], 1517 | config["storage"]["ssh-key"], 1518 | verify_ssl=not args.disable_verify_ssl, 1519 | ) 1520 | storage_project, storage_package = pathlib.Path( 1521 | config["storage"]["storage"] 1522 | ).parts 1523 | 1524 | if not await storage_obs.authorized(storage_project, storage_package): 1525 | print("No authorization to access the file storage in build service") 1526 | sys.exit(-1) 1527 | 1528 | if not await storage_obs.exists(storage_project, storage_package): 1529 | print("File storage not found in build service") 1530 | sys.exit(-1) 1531 | 1532 | storage = await StorageOBS(storage_obs, storage_project, storage_package, git) 1533 | elif storage_type == "lfs": 1534 | storage = StorageLFS(git) 1535 | 1536 | if not await storage.is_installed(): 1537 | print("LFS extension not installed") 1538 | sys.exit(-1) 1539 | print("Git LFS extension enabled in the repository") 1540 | else: 1541 | raise NotImplementedError(f"Storage {storage_type} not implemented") 1542 | 1543 | importer = Importer( 1544 | obs, 1545 | git, 1546 | storage, 1547 | args.remove_role_project_meta, 1548 | args.skip_project_meta, 1549 | args.skip_all_project_meta, 1550 | args.remove_role_package_meta, 1551 | args.skip_package_meta, 1552 | args.skip_all_package_meta, 1553 | args.skip_changes_commit_hash, 1554 | ) 1555 | 1556 | if args.adjust_release: 1557 | importer.load_revisions(args.adjust_release) 1558 | 1559 | if package: 1560 | # If the project is not present, maybe we want to create it 1561 | if not (await obs.exists(project) or args.skip_all_project_meta): 1562 | await importer.project_metadata(project) 1563 | await importer.package(project, package) 1564 | else: 1565 | await importer.project(project) 1566 | 1567 | if storage_type == "obs": 1568 | await storage_obs.close() 1569 | await obs.close() 1570 | 1571 | 1572 | def main(): 1573 | parser = argparse.ArgumentParser(description="OBS-git simple bridge tool") 1574 | parser.add_argument( 1575 | "--config", 1576 | "-c", 1577 | type=pathlib.Path, 1578 | default=pathlib.Path("~", ".obsgit").expanduser(), 1579 | help="configuration file", 1580 | ) 1581 | parser.add_argument( 1582 | "--level", 1583 | "-l", 1584 | help="logging level", 1585 | ) 1586 | parser.add_argument( 1587 | "--disable-verify-ssl", 1588 | action="store_true", 1589 | help="disable SSL verification", 1590 | ) 1591 | 1592 | subparser = parser.add_subparsers() 1593 | 1594 | parser_create_config = subparser.add_parser( 1595 | "create-config", help="create default config file" 1596 | ) 1597 | parser_create_config.add_argument( 1598 | "--api", 1599 | "-a", 1600 | default="https://api.opensuse.org", 1601 | help="url for the api", 1602 | ) 1603 | parser_create_config.add_argument( 1604 | "--username", 1605 | "-u", 1606 | default=getpass.getuser(), 1607 | help="username for login", 1608 | ) 1609 | parser_create_config.add_argument( 1610 | "--password", 1611 | "-p", 1612 | help="password for login or SSH key passphrase", 1613 | ) 1614 | parser_create_config.add_argument( 1615 | "--ssh-key", 1616 | "-k", 1617 | type=pathlib.Path, 1618 | help="SSH key file for login", 1619 | ) 1620 | parser_create_config.add_argument( 1621 | "--link", 1622 | "-l", 1623 | choices=["never", "always", "auto"], 1624 | default="never", 1625 | help="expand package links", 1626 | ) 1627 | parser_create_config.add_argument( 1628 | "--storage", 1629 | "-s", 1630 | choices=["obs", "lfs"], 1631 | default="lfs", 1632 | help="type of storage for large files", 1633 | ) 1634 | parser_create_config.add_argument( 1635 | "--prefix", 1636 | default="packages", 1637 | help="git directory where all the packages will be stored", 1638 | ) 1639 | parser_create_config.set_defaults(func=create_config) 1640 | 1641 | parser_export = subparser.add_parser("export", help="export between OBS and git") 1642 | parser_export.add_argument("project", help="OBS project name") 1643 | parser_export.add_argument( 1644 | "repository", nargs="?", default=".", help="git repository directory" 1645 | ) 1646 | parser_export.add_argument("--package", "-p", help="OBS package name") 1647 | parser_export.add_argument( 1648 | "--skip-project-meta", 1649 | action="store_true", 1650 | help="skip update project _meta", 1651 | ) 1652 | parser_export.add_argument( 1653 | "--skip-all-project-meta", 1654 | action="store_true", 1655 | help="skip update all project metadata", 1656 | ) 1657 | parser_export.add_argument( 1658 | "--skip-package-meta", 1659 | action="store_true", 1660 | help="skip update package _meta", 1661 | ) 1662 | parser_export.add_argument( 1663 | "--skip-all-package-meta", 1664 | action="store_true", 1665 | help="skip update all package metadata", 1666 | ) 1667 | parser_export.add_argument( 1668 | "--only-export-revisions", 1669 | type=pathlib.Path, 1670 | metavar="REVISION.CSV", 1671 | help="only export the revision numbers from OBS", 1672 | ) 1673 | parser_export.set_defaults(func=export) 1674 | 1675 | parser_import = subparser.add_parser("import", help="import between git and OBS") 1676 | parser_import.add_argument( 1677 | "repository", nargs="?", default=".", help="git repository directory" 1678 | ) 1679 | parser_import.add_argument("project", help="OBS project name") 1680 | parser_import.add_argument("--package", "-p", help="OBS package name") 1681 | parser_import.add_argument( 1682 | "--remove-role-project-meta", 1683 | action="store_true", 1684 | help="remove <person> and <group> from project _meta", 1685 | ) 1686 | parser_import.add_argument( 1687 | "--skip-project-meta", 1688 | action="store_true", 1689 | help="skip update project _meta", 1690 | ) 1691 | parser_import.add_argument( 1692 | "--skip-all-project-meta", 1693 | action="store_true", 1694 | help="skip update all project metadata", 1695 | ) 1696 | parser_import.add_argument( 1697 | "--remove-role-package-meta", 1698 | action="store_true", 1699 | help="remove <person> and <group> from package _meta", 1700 | ) 1701 | parser_import.add_argument( 1702 | "--skip-package-meta", 1703 | action="store_true", 1704 | help="skip update package _meta", 1705 | ) 1706 | parser_import.add_argument( 1707 | "--skip-all-package-meta", 1708 | action="store_true", 1709 | help="skip update all package metadata", 1710 | ) 1711 | parser_import.add_argument( 1712 | "--adjust-release", 1713 | type=pathlib.Path, 1714 | metavar="REVISION.CSV", 1715 | help="adjust the release based on the revision history", 1716 | ) 1717 | parser_import.add_argument( 1718 | "--skip-changes-commit-hash", 1719 | action="store_true", 1720 | help="do not prepend .changes files with latest git commit hash", 1721 | ) 1722 | 1723 | parser_import.set_defaults(func=import_) 1724 | 1725 | args = parser.parse_args() 1726 | 1727 | if args.level: 1728 | numeric_level = getattr(logging, args.level.upper(), None) 1729 | if not isinstance(numeric_level, int): 1730 | print(f"Invalid log level: {args.level}") 1731 | sys.exit(-1) 1732 | logging.basicConfig(level=numeric_level) 1733 | 1734 | if "func" not in args: 1735 | parser.print_help() 1736 | sys.exit(-1) 1737 | 1738 | if args.func == create_config: 1739 | args.func(args) 1740 | else: 1741 | config = read_config(args.config) 1742 | # TODO: For Python >= 3.7 use get_running_loop() 1743 | loop = asyncio.get_event_loop() 1744 | loop.run_until_complete(args.func(args, config)) 1745 | loop.close() 1746 | 1747 | 1748 | if __name__ == "__main__": 1749 | main() 1750 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | . 2 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import setuptools 2 | 3 | with open("README.md", "r") as f: 4 | long_description = f.read() 5 | 6 | 7 | setuptools.setup( 8 | name="obsgit", 9 | version="0.1.0", 10 | author="Alberto Planas", 11 | author_email="aplanas@gmail.com", 12 | description="Simple bridge between Open Build Server and Git", 13 | long_description=long_description, 14 | long_description_content_type="text/markdown", 15 | url="https://github.com/aplanas/obsgit", 16 | packages=setuptools.find_packages(), 17 | python_requires=">=3.6", 18 | classifiers=[ 19 | "Development Status :: 5 - Production/Stable", 20 | "Environment :: Console", 21 | "Intended Audience :: Developers", 22 | "Intended Audience :: Information Technology", 23 | "Intended Audience :: System Administrators", 24 | "License :: OSI Approved :: Apache Software License", 25 | "Programming Language :: Python :: 3", 26 | "Topic :: Software Development :: Build Tools", 27 | "Topic :: System :: Archiving :: Packaging", 28 | ], 29 | entry_points={ 30 | "console_scripts": ["obsgit=obsgit.obsgit:main"], 31 | }, 32 | install_requires=[ 33 | "aiohttp", 34 | "chardet", 35 | "pygit2", 36 | ], 37 | ) 38 | -------------------------------------------------------------------------------- /tests/test_obsgit.py: -------------------------------------------------------------------------------- 1 | import contextlib 2 | import os 3 | import pathlib 4 | import tempfile 5 | import unittest 6 | import unittest.mock 7 | import xml.etree.ElementTree as ET 8 | 9 | from obsgit import obsgit 10 | 11 | 12 | class TestReadConfig(unittest.TestCase): 13 | config_filename = pathlib.Path("/tmp/config") 14 | 15 | def setUp(self): 16 | self._remove_config_filename() 17 | 18 | def tearDown(self): 19 | self._remove_config_filename() 20 | 21 | def _remove_config_filename(self): 22 | try: 23 | self.config_filename.unlink() 24 | except FileNotFoundError: 25 | pass 26 | 27 | def test_default_content(self): 28 | with open(os.devnull, "w") as devnull: 29 | with contextlib.redirect_stdout(devnull): 30 | config = obsgit.read_config(self.config_filename) 31 | self.assertEqual(config["import"]["url"], "https://api.opensuse.org") 32 | self.assertEqual(config["import"]["username"], os.getlogin()) 33 | self.assertEqual(config["import"]["password"], "password") 34 | self.assertEqual(config["export"]["url"], "https://api.opensuse.org") 35 | self.assertEqual(config["export"]["username"], os.getlogin()) 36 | self.assertEqual(config["export"]["password"], "password") 37 | self.assertEqual( 38 | config["export"]["storage"], f"home:{os.getlogin()}:storage/files" 39 | ) 40 | 41 | def test_default_content_when_url(self): 42 | with open(os.devnull, "w") as devnull: 43 | with contextlib.redirect_stdout(devnull): 44 | config = obsgit.read_config( 45 | self.config_filename, url="https://api.suse.de" 46 | ) 47 | self.assertEqual(config["import"]["url"], "https://api.suse.de") 48 | self.assertEqual(config["import"]["username"], os.getlogin()) 49 | self.assertEqual(config["import"]["password"], "password") 50 | self.assertEqual(config["export"]["url"], "https://api.suse.de") 51 | self.assertEqual(config["export"]["username"], os.getlogin()) 52 | self.assertEqual(config["export"]["password"], "password") 53 | self.assertEqual( 54 | config["export"]["storage"], f"home:{os.getlogin()}:storage/files" 55 | ) 56 | 57 | def test_default_content_when_username(self): 58 | with open(os.devnull, "w") as devnull: 59 | with contextlib.redirect_stdout(devnull): 60 | config = obsgit.read_config(self.config_filename, username="user") 61 | self.assertEqual(config["import"]["url"], "https://api.opensuse.org") 62 | self.assertEqual(config["import"]["username"], "user") 63 | self.assertEqual(config["import"]["password"], "password") 64 | self.assertEqual(config["export"]["url"], "https://api.opensuse.org") 65 | self.assertEqual(config["export"]["username"], "user") 66 | self.assertEqual(config["export"]["password"], "password") 67 | self.assertEqual(config["export"]["storage"], f"home:user:storage/files") 68 | 69 | def test_default_content_when_password(self): 70 | with open(os.devnull, "w") as devnull: 71 | with contextlib.redirect_stdout(devnull): 72 | config = obsgit.read_config(self.config_filename, password="secret") 73 | self.assertEqual(config["import"]["url"], "https://api.opensuse.org") 74 | self.assertEqual(config["import"]["username"], os.getlogin()) 75 | self.assertEqual(config["import"]["password"], "secret") 76 | self.assertEqual(config["export"]["url"], "https://api.opensuse.org") 77 | self.assertEqual(config["export"]["username"], os.getlogin()) 78 | self.assertEqual(config["export"]["password"], "secret") 79 | self.assertEqual( 80 | config["export"]["storage"], f"home:{os.getlogin()}:storage/files" 81 | ) 82 | 83 | def test_default_persmissions(self): 84 | with open(os.devnull, "w") as devnull: 85 | with contextlib.redirect_stdout(devnull): 86 | obsgit.read_config(self.config_filename) 87 | self.assertTrue(self.config_filename.exists()) 88 | self.assertEqual(self.config_filename.stat().st_mode, 33152) 89 | 90 | def test_custom_content(self): 91 | with open(self.config_filename, "w") as f: 92 | f.write( 93 | """ 94 | [import] 95 | url = https://api.import.com 96 | username = user_import 97 | password = passwd_import 98 | 99 | [export] 100 | url = https://api.export.com 101 | username = user_export 102 | password = passwd_export 103 | storage = project:storage/files 104 | """ 105 | ) 106 | config = obsgit.read_config(self.config_filename) 107 | self.assertEqual(config["import"]["url"], "https://api.import.com") 108 | self.assertEqual(config["import"]["username"], "user_import") 109 | self.assertEqual(config["import"]["password"], "passwd_import") 110 | self.assertEqual(config["export"]["url"], "https://api.export.com") 111 | self.assertEqual(config["export"]["username"], "user_export") 112 | self.assertEqual(config["export"]["password"], "passwd_export") 113 | self.assertEqual(config["export"]["storage"], f"project:storage/files") 114 | 115 | 116 | class TestAsyncOBS(unittest.IsolatedAsyncioTestCase): 117 | @unittest.mock.patch.object(obsgit, "aiohttp") 118 | def test_open(self, aiohttp): 119 | obs = obsgit.AsyncOBS("https://api.example.local", "user", "secret") 120 | self.assertEqual(obs.url, "https://api.example.local") 121 | self.assertEqual(obs.username, "user") 122 | aiohttp.BasicAuth.assert_called_once_with("user", "secret") 123 | self.assertNotEqual(obs.client, None) 124 | 125 | async def test_close(self): 126 | obs = obsgit.AsyncOBS("https://api.example.local", "user", "secret") 127 | await obs.close() 128 | self.assertEqual(obs.client, None) 129 | 130 | async def test_create_enabled_project(self): 131 | obs = obsgit.AsyncOBS("https://api.example.local", "user", "secret") 132 | with unittest.mock.patch.object(obs, "authorized", return_value=True): 133 | with unittest.mock.patch.object(obs, "exists", return_value=False): 134 | with unittest.mock.patch.object( 135 | obs, "client", new_callable=unittest.mock.AsyncMock 136 | ) as client: 137 | await obs.create("myproject") 138 | client.put.assert_called_once_with( 139 | "https://api.example.local/source/myproject/_meta", 140 | data=( 141 | '<project name="myproject"><title/><description/>' 142 | '<person userid="user" role="maintainer"/></project>' 143 | ), 144 | ) 145 | await obs.close() 146 | 147 | async def test_create_disabled_project(self): 148 | obs = obsgit.AsyncOBS("https://api.example.local", "user", "secret") 149 | with unittest.mock.patch.object(obs, "authorized", return_value=True): 150 | with unittest.mock.patch.object(obs, "exists", return_value=False): 151 | with unittest.mock.patch.object( 152 | obs, "client", new_callable=unittest.mock.AsyncMock 153 | ) as client: 154 | await obs.create("myproject", disabled=True) 155 | client.put.assert_called_once_with( 156 | "https://api.example.local/source/myproject/_meta", 157 | data=( 158 | '<project name="myproject"><title/><description/>' 159 | '<person userid="user" role="maintainer"/><build>' 160 | "<disable/></build><publish><disable/></publish>" 161 | "<useforbuild><disable/></useforbuild></project>" 162 | ), 163 | ) 164 | await obs.close() 165 | 166 | async def test_create_non_authorized_project(self): 167 | obs = obsgit.AsyncOBS("https://api.example.local", "user", "secret") 168 | with unittest.mock.patch.object(obs, "authorized", return_value=False): 169 | with unittest.mock.patch.object(obs, "exists", return_value=False): 170 | with unittest.mock.patch.object( 171 | obs, "client", new_callable=unittest.mock.AsyncMock 172 | ) as client: 173 | await obs.create("myproject") 174 | client.put.assert_not_called() 175 | await obs.close() 176 | 177 | async def test_create_existent_project(self): 178 | obs = obsgit.AsyncOBS("https://api.example.local", "user", "secret") 179 | with unittest.mock.patch.object(obs, "authorized", return_value=True): 180 | with unittest.mock.patch.object(obs, "exists", return_value=True): 181 | with unittest.mock.patch.object( 182 | obs, "client", new_callable=unittest.mock.AsyncMock 183 | ) as client: 184 | await obs.create("myproject") 185 | client.put.assert_not_called() 186 | await obs.close() 187 | 188 | async def test_create_enabled_package(self): 189 | obs = obsgit.AsyncOBS("https://api.example.local", "user", "secret") 190 | with unittest.mock.patch.object(obs, "authorized", return_value=True): 191 | with unittest.mock.patch.object(obs, "exists", side_effect=[True, False]): 192 | with unittest.mock.patch.object( 193 | obs, "client", new_callable=unittest.mock.AsyncMock 194 | ) as client: 195 | await obs.create("myproject", "mypackage") 196 | client.put.assert_called_once_with( 197 | "https://api.example.local/source/myproject/mypackage/_meta", 198 | data=( 199 | '<package name="mypackage" project="myproject">' 200 | "<title/><description/></package>" 201 | ), 202 | ) 203 | await obs.close() 204 | 205 | async def test_create_disabled_package(self): 206 | obs = obsgit.AsyncOBS("https://api.example.local", "user", "secret") 207 | with unittest.mock.patch.object(obs, "authorized", return_value=True): 208 | with unittest.mock.patch.object(obs, "exists", side_effect=[True, False]): 209 | with unittest.mock.patch.object( 210 | obs, "client", new_callable=unittest.mock.AsyncMock 211 | ) as client: 212 | await obs.create("myproject", "mypackage", disabled=True) 213 | client.put.assert_called_once_with( 214 | "https://api.example.local/source/myproject/mypackage/_meta", 215 | data=( 216 | '<package name="mypackage" project="myproject"><title/>' 217 | "<description/><build><disable/></build><publish><disable/>" 218 | "</publish><useforbuild><disable/></useforbuild></package>" 219 | ), 220 | ) 221 | await obs.close() 222 | 223 | async def test_download(self): 224 | obs = obsgit.AsyncOBS("https://api.example.local", "user", "secret") 225 | with unittest.mock.patch.object(obs, "_download") as download: 226 | await obs.download( 227 | "myproject", 228 | "mypackage", 229 | "myfile", 230 | filename_path="filename", 231 | params=[("rev", "latest")], 232 | ) 233 | download.assert_called_once_with( 234 | "source/myproject/mypackage/myfile", 235 | "filename", 236 | params=[("rev", "latest")], 237 | ) 238 | await obs.close() 239 | 240 | async def test_upload(self): 241 | obs = obsgit.AsyncOBS("https://api.example.local", "user", "secret") 242 | with unittest.mock.patch.object(obs, "_upload") as upload: 243 | await obs.upload( 244 | "myproject", "mypackage", "myfile", filename_path="filename", 245 | ) 246 | upload.assert_called_once_with( 247 | "source/myproject/mypackage/myfile", 248 | filename_path="filename", 249 | data=None, 250 | params=None, 251 | ) 252 | await obs.close() 253 | 254 | async def test_delete(self): 255 | obs = obsgit.AsyncOBS("https://api.example.local", "user", "secret") 256 | with unittest.mock.patch.object(obs, "_delete") as delete: 257 | await obs.delete("myproject", "mypackage", "myfile") 258 | delete.assert_called_once_with( 259 | "source/myproject/mypackage/myfile", params=None, 260 | ) 261 | await obs.close() 262 | 263 | async def test_transfer(self): 264 | obs = obsgit.AsyncOBS("https://api.example.local", "user", "secret") 265 | with unittest.mock.patch.object(obs, "_transfer") as transfer: 266 | await obs.transfer("myproject", "mypackage", "myfile", "to_myproject") 267 | transfer.assert_called_once_with( 268 | "source/myproject/mypackage/myfile", 269 | "source/to_myproject/mypackage/myfile", 270 | None, 271 | None, 272 | ) 273 | await obs.close() 274 | 275 | async def test_packages(self): 276 | obs = obsgit.AsyncOBS("https://api.example.local", "user", "secret") 277 | with unittest.mock.patch.object(obs, "_xml") as xml: 278 | xml.return_value = ET.fromstring( 279 | '<directory count="2"><entry name="package1"/>' 280 | '<entry name="package2"/></directory>' 281 | ) 282 | packages = await obs.packages("myproject") 283 | self.assertEqual(packages, ["package1", "package2"]) 284 | await obs.close() 285 | 286 | async def test_files_md5_revision(self): 287 | obs = obsgit.AsyncOBS("https://api.example.local", "user", "secret") 288 | with unittest.mock.patch.object(obs, "_xml") as xml: 289 | xml.return_value = ET.fromstring( 290 | '<directory name="mypackage" rev="5" vrev="5" srcmd5="srcmd5">' 291 | '<entry name="file1" md5="md51" size="1024" mtime="1234567890"/>' 292 | '<entry name="file2" md5="md52" size="1024" mtime="1234567890"/>' 293 | "</directory>" 294 | ) 295 | files_md5, revision = await obs.files_md5_revision("myproject", "mypackage") 296 | self.assertEqual(files_md5, [("file1", "md51"), ("file2", "md52")]) 297 | self.assertEqual(revision, "5") 298 | await obs.close() 299 | 300 | async def test_files_md5_revision_linkinfo(self): 301 | obs = obsgit.AsyncOBS("https://api.example.local", "user", "secret") 302 | with unittest.mock.patch.object(obs, "_xml") as xml: 303 | xml.side_effect = [ 304 | ET.fromstring( 305 | '<directory name="mypackage" rev="4" vrev="4" srcmd5="srcmd51">' 306 | '<linkinfo project="myproject" package="mypackage" srcmd5="srcmd51"' 307 | ' baserev="baserev1" xsrcmd5="xsrcmd51" lsrcmd5="lsrcmd51"/>' 308 | '<entry name="_link" md5="md50" size="1024" mtime="1234567890"/>' 309 | '<entry name="file1" md5="md51" size="1024" mtime="1234567890"/>' 310 | "</directory>" 311 | ), 312 | ET.fromstring( 313 | '<directory name="mypackage" rev="5" vrev="5" srcmd5="srcmd52">' 314 | '<linkinfo project="myproject" package="mypackage" srcmd5="srcmd52"' 315 | ' baserev="baserev2" xsrcmd5="xsrcmd52" lsrcmd5="lsrcmd52"/>' 316 | '<entry name="file1" md5="md51" size="1024" mtime="1234567890"/>' 317 | '<entry name="file2" md5="md52" size="1024" mtime="1234567890"/>' 318 | "</directory>" 319 | ), 320 | ] 321 | files_md5, revision = await obs.files_md5_revision("myproject", "mypackage") 322 | self.assertEqual(files_md5, [("file1", "md51"), ("file2", "md52")]) 323 | self.assertEqual(revision, "xsrcmd51") 324 | await obs.close() 325 | 326 | 327 | class TestGit(unittest.IsolatedAsyncioTestCase): 328 | def test_exists_and_create(self): 329 | with tempfile.TemporaryDirectory() as tmp: 330 | git = obsgit.Git(tmp) 331 | self.assertFalse(git.exists()) 332 | git.create() 333 | self.assertTrue(git.exists()) 334 | 335 | async def test_delete_package(self): 336 | with tempfile.TemporaryDirectory() as tmp: 337 | tmp = pathlib.Path(tmp) 338 | git = obsgit.Git(tmp) 339 | 340 | package_path = tmp / "mypackage" 341 | package_path.mkdir() 342 | 343 | self.assertTrue(package_path.exists()) 344 | await git.delete("mypackage") 345 | self.assertFalse(package_path.exists()) 346 | 347 | async def test_delete_filename(self): 348 | with tempfile.TemporaryDirectory() as tmp: 349 | tmp = pathlib.Path(tmp) 350 | git = obsgit.Git(tmp) 351 | 352 | package_path = tmp / "mypackage" 353 | package_path.mkdir() 354 | 355 | filename_path = package_path / "myfile" 356 | filename_path.touch() 357 | 358 | self.assertTrue(package_path.exists()) 359 | self.assertTrue(filename_path.exists()) 360 | await git.delete("mypackage", "myfile") 361 | self.assertTrue(package_path.exists()) 362 | self.assertFalse(filename_path.exists()) 363 | 364 | async def test_packages(self): 365 | with tempfile.TemporaryDirectory() as tmp: 366 | tmp = pathlib.Path(tmp) 367 | git = obsgit.Git(tmp) 368 | 369 | for package in ("mypackage1", "mypackage2", ".git", ".obs"): 370 | (tmp / package).mkdir() 371 | 372 | self.assertEqual(git.packages(), ["mypackage1", "mypackage2"]) 373 | 374 | async def test_files_md5(self): 375 | with tempfile.TemporaryDirectory() as tmp: 376 | tmp = pathlib.Path(tmp) 377 | git = obsgit.Git(tmp) 378 | 379 | package_path = tmp / "mypackage" 380 | package_path.mkdir() 381 | 382 | for filename in ("myfile1", "myfile2"): 383 | with (package_path / filename).open("w") as f: 384 | f.write(filename) 385 | 386 | self.assertEqual( 387 | list(await git.files_md5("mypackage")), 388 | [ 389 | ("myfile1", "52a082e3940c1bda8306223103eaab28"), 390 | ("myfile2", "549d8b648caf7cce417751c0fbe15c7a"), 391 | ], 392 | ) 393 | 394 | 395 | class TestStorage(unittest.IsolatedAsyncioTestCase): 396 | async def test_storage(self): 397 | obs = obsgit.AsyncOBS("https://api.example.local", "user", "secret") 398 | with unittest.mock.patch.object( 399 | obs, "files_md5_revision" 400 | ) as obs_files_md5_revision: 401 | obs_files_md5_revision.return_value = ( 402 | [("md51", "md51"), ("md52", "md52")], 403 | None, 404 | ) 405 | storage = await obsgit.Storage(obs, "project/package") 406 | 407 | self.assertEqual(storage.project, "project") 408 | self.assertEqual(storage.package, "package") 409 | self.assertEqual(storage.index, {"md51", "md52"}) 410 | await obs.close() 411 | 412 | async def test_transfer(self): 413 | obs = obsgit.AsyncOBS("https://api.example.local", "user", "secret") 414 | with unittest.mock.patch.object( 415 | obs, "files_md5_revision" 416 | ) as obs_files_md5_revision: 417 | obs_files_md5_revision.return_value = ( 418 | [("md51", "md51"), ("md52", "md52")], 419 | None, 420 | ) 421 | storage = await obsgit.Storage(obs, "project/package") 422 | 423 | with unittest.mock.patch.object(obs, "transfer") as obs_transfer: 424 | await storage.transfer("md51", "myproject", "mypackage", "myfile", obs) 425 | obs_transfer.assert_called_once_with( 426 | "project", "package", "md51", "myproject", "mypackage", "myfile", obs 427 | ) 428 | 429 | await obs.close() 430 | 431 | 432 | class TestExporterIsBinary(unittest.TestCase): 433 | unknown_filename = pathlib.Path("/tmp/unknown") 434 | 435 | def setUp(self): 436 | self._remove_unknown_filename() 437 | 438 | def tearDown(self): 439 | self._remove_unknown_filename() 440 | 441 | def _remove_unknown_filename(self): 442 | try: 443 | self.unknown_filename.unlink() 444 | except FileNotFoundError: 445 | pass 446 | 447 | def test_is_binary_shortcut(self): 448 | self.assertTrue(obsgit.Exporter.is_binary("foo.tar.gz")) 449 | 450 | def test_is_non_binary_shorcut(self): 451 | self.assertFalse(obsgit.Exporter.is_binary("foo.spec")) 452 | 453 | def test_is_non_binary_exception_shorcut(self): 454 | self.assertTrue(obsgit.Exporter.is_binary("foo.obscpio")) 455 | 456 | def test_is_binary(self): 457 | with open(self.unknown_filename, "wb") as f: 458 | f.write(b"MZ\xea\x07\x00\xc0\x07\x8c") 459 | self.assertTrue(obsgit.Exporter.is_binary(self.unknown_filename)) 460 | 461 | def test_is_non_binary(self): 462 | with open(self.unknown_filename, "w") as f: 463 | f.write("some text") 464 | self.assertFalse(obsgit.Exporter.is_binary(self.unknown_filename)) 465 | 466 | 467 | class TestExporter(unittest.IsolatedAsyncioTestCase): 468 | async def asyncSetUp(self): 469 | self.obs = obsgit.AsyncOBS("https://api.example.local", "user", "secret") 470 | with unittest.mock.patch.object( 471 | self.obs, "files_md5_revision", return_value=[(), None] 472 | ): 473 | self.storage = await obsgit.Storage(self.obs, "project/package") 474 | self.git = obsgit.Git("/tmp/git") 475 | self.exporter = obsgit.Exporter(self.obs, self.git, self.storage) 476 | 477 | async def asyncTearDown(self): 478 | await self.obs.close() 479 | 480 | async def test_project(self): 481 | packages_obs = ["package1", "package2"] 482 | packages_git = ["package2", "package3"] 483 | self.obs.packages = unittest.mock.AsyncMock(return_value=packages_obs) 484 | self.git.packages = unittest.mock.MagicMock(return_value=packages_git) 485 | 486 | self.exporter.project_metadata = unittest.mock.AsyncMock() 487 | self.exporter.package = unittest.mock.AsyncMock() 488 | self.git.delete = unittest.mock.AsyncMock() 489 | 490 | await self.exporter.project("myproject") 491 | 492 | self.exporter.project_metadata.assert_called_once_with("myproject") 493 | self.exporter.package.assert_has_calls( 494 | [ 495 | unittest.mock.call("myproject", "package1"), 496 | unittest.mock.call("myproject", "package2"), 497 | ], 498 | any_order=True, 499 | ) 500 | self.git.delete.assert_called_once_with("package3") 501 | 502 | async def test_project_metadata(self): 503 | self.obs.download = unittest.mock.AsyncMock() 504 | 505 | with tempfile.TemporaryDirectory() as tmp: 506 | tmp = pathlib.Path(tmp) 507 | self.git.path = tmp 508 | 509 | await self.exporter.project_metadata("myproject") 510 | 511 | self.assertTrue((tmp / ".obs").exists()) 512 | 513 | self.obs.download.assert_has_calls( 514 | [ 515 | unittest.mock.call( 516 | "myproject", "_meta", filename_path=(tmp / ".obs" / "_meta") 517 | ), 518 | unittest.mock.call( 519 | "myproject", "_project", filename_path=(tmp / ".obs" / "_project") 520 | ), 521 | unittest.mock.call( 522 | "myproject", 523 | "_attribute", 524 | filename_path=(tmp / ".obs" / "_attribute") 525 | ), 526 | unittest.mock.call( 527 | "myproject", "_config", filename_path=(tmp / ".obs" / "_config") 528 | ), 529 | unittest.mock.call( 530 | "myproject", "_pattern", filename_path=(tmp / ".obs" / "_pattern") 531 | ), 532 | ], 533 | any_order=True, 534 | ) 535 | 536 | async def test_package(self): 537 | files_md5_obs = ( 538 | [("file1", "md51"), ("file2", "md52"), ("file3", "md531")], 539 | "revision", 540 | ) 541 | files_md5_git = [("file2", "md52"), ("file3", "md532"), ("file4", "md54")] 542 | store_index = {"md52"} 543 | is_binary = {"file1", "file2"} 544 | 545 | self.obs.files_md5_revision = unittest.mock.AsyncMock( 546 | return_value=files_md5_obs 547 | ) 548 | self.git.files_md5 = unittest.mock.AsyncMock(return_value=files_md5_git) 549 | self.storage.index = store_index 550 | 551 | self.exporter.package_metadata = unittest.mock.AsyncMock() 552 | self.obs.download = unittest.mock.AsyncMock() 553 | self.obs.upload = unittest.mock.AsyncMock() 554 | self.git.delete = unittest.mock.AsyncMock() 555 | 556 | with unittest.mock.patch.object( 557 | obsgit.Exporter, 558 | "is_binary", 559 | side_effect=lambda x: x.parts[-1] in is_binary, 560 | ) as exporter_is_binary: 561 | with tempfile.TemporaryDirectory() as tmp: 562 | tmp = pathlib.Path(tmp) 563 | self.git.path = tmp 564 | (tmp / "mypackage" / ".obs").mkdir(parents=True) 565 | 566 | await self.exporter.package("myproject", "mypackage") 567 | 568 | self.assertTrue((self.git.path / "mypackage").exists()) 569 | self.exporter.package_metadata.assert_called_once_with( 570 | "myproject", "mypackage" 571 | ) 572 | self.obs.download.assert_has_calls( 573 | [ 574 | unittest.mock.call( 575 | "myproject", 576 | "mypackage", 577 | "file1", 578 | filename_path=tmp / "mypackage" / "file1", 579 | params=[("rev", "revision")], 580 | ), 581 | unittest.mock.call( 582 | "myproject", 583 | "mypackage", 584 | "file3", 585 | filename_path=tmp / "mypackage" / "file3", 586 | params=[("rev", "revision")], 587 | ), 588 | ], 589 | any_order=True, 590 | ) 591 | self.git.delete.assert_has_calls( 592 | [ 593 | unittest.mock.call("mypackage", "file4"), 594 | # Remove because is a binary file 595 | unittest.mock.call("mypackage", "file1"), 596 | ] 597 | ) 598 | exporter_is_binary.assert_has_calls( 599 | [ 600 | unittest.mock.call(tmp / "mypackage" / "file1"), 601 | unittest.mock.call(tmp / "mypackage" / "file3"), 602 | ], 603 | any_order=True, 604 | ) 605 | self.obs.upload.assert_called_once_with( 606 | "project", 607 | "package", 608 | "md51", 609 | filename_path=(tmp / "mypackage" / "file1"), 610 | ) 611 | with (tmp / "mypackage" / ".obs" / "files").open() as files: 612 | self.assertEqual(files.read(), "file1\t\tmd51\nfile2\t\tmd52\n") 613 | 614 | async def test_package_metadata(self): 615 | self.obs.download = unittest.mock.AsyncMock() 616 | 617 | with tempfile.TemporaryDirectory() as tmp: 618 | tmp = pathlib.Path(tmp) 619 | self.git.path = tmp 620 | (tmp / "mypackage").mkdir() 621 | 622 | await self.exporter.package_metadata("myproject", "mypackage") 623 | 624 | self.assertTrue((tmp / "mypackage" / ".obs").exists()) 625 | 626 | self.obs.download.assert_has_calls( 627 | [ 628 | unittest.mock.call( 629 | "myproject", 630 | "mypackage", 631 | "_meta", 632 | filename_path=(tmp / "mypackage" / ".obs" / "_meta"), 633 | ), 634 | unittest.mock.call( 635 | "myproject", 636 | "mypackage", 637 | "_attribute", 638 | filename_path=(tmp / "mypackage" / ".obs" / "_attribute"), 639 | ), 640 | unittest.mock.call( 641 | "myproject", 642 | "mypackage", 643 | "_history", 644 | filename_path=(tmp / "mypackage" / ".obs" / "_history"), 645 | ), 646 | ], 647 | any_order=True, 648 | ) 649 | 650 | 651 | if __name__ == "__main__": 652 | unittest.main() 653 | --------------------------------------------------------------------------------