├── .appveyor.yml ├── .gitignore ├── .travis.yml ├── CHANGES.rst ├── LICENSE.txt ├── MANIFEST.in ├── Makefile ├── README.rst ├── appveyor-requirements.txt ├── git_archive_all.py ├── git_archive_all.pyi ├── py.typed ├── setup.cfg ├── setup.py ├── test-requirements.txt ├── test_git_archive_all.py └── travis-requirements.txt /.appveyor.yml: -------------------------------------------------------------------------------- 1 | clone_depth: 1 2 | 3 | image: 4 | - Visual Studio 2019 5 | 6 | environment: 7 | PYTHON: "C:\\Python37-x64" 8 | 9 | matrix: 10 | # Python 2.x builds do not work because implementation of os.readlink did not support Windows till 3.2 11 | - TOXENV: "py34" 12 | - TOXENV: "py35" 13 | - TOXENV: "py36" 14 | - TOXENV: "py37" 15 | 16 | matrix: 17 | fast_finish: true 18 | 19 | build: off 20 | 21 | install: 22 | - "%PYTHON%\\python.exe -m pip install --upgrade pip" 23 | - "%PYTHON%\\python.exe -m pip install --upgrade wheel>=0.30.0 setuptools>=36.6.0" 24 | - "%PYTHON%\\python.exe -m pip install -r appveyor-requirements.txt" 25 | 26 | test_script: 27 | - "%PYTHON%\\python.exe -m tox -vv -- --cov-report=xml" 28 | 29 | after_test: 30 | - "%PYTHON%\\python.exe -m codecov --required --file %APPVEYOR_BUILD_FOLDER%\\coverage.xml" 31 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Textmate - if you build your xcode projects with it 2 | *.tm_build_errors 3 | 4 | # osx noise 5 | .DS_Store 6 | profile 7 | 8 | # emacs noise 9 | *~ -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | git: 2 | depth: 1 3 | 4 | cache: 5 | directories: 6 | - $HOME/.cache/pyenv 7 | - /opt/pyenv/versions/$PYENV_VERSION 8 | - $HOME/pyenv/versions/$PYENV_VERSION 9 | pip: true 10 | 11 | env: 12 | global: 13 | - PYTHON_BUILD_CACHE_PATH=$HOME/.cache/pyenv 14 | 15 | addons: 16 | homebrew: 17 | packages: 18 | - pyenv 19 | - openssl 20 | - readline 21 | - xz 22 | - zlib 23 | update: true 24 | 25 | jobs: 26 | fast_finish: true 27 | include: 28 | - &linux 29 | os: linux 30 | language: python 31 | env: PYENV_VERSION=2.6.9 TOXENV=py26 32 | - <<: *linux 33 | env: PYENV_VERSION=2.7.18 TOXENV=py27 34 | - <<: *linux 35 | env: PYENV_VERSION=2.7-dev TOXENV=py27 36 | - <<: *linux 37 | env: PYENV_VERSION=3.4.10 TOXENV=py34 38 | - <<: *linux 39 | env: PYENV_VERSION=3.5.10 TOXENV=py35 40 | - <<: *linux 41 | env: PYENV_VERSION=3.5-dev TOXENV=py35 42 | - <<: *linux 43 | env: PYENV_VERSION=3.6.12 TOXENV=py36 DEPLOY=1 44 | - <<: *linux 45 | env: PYENV_VERSION=3.6-dev TOXENV=py36 46 | - <<: *linux 47 | env: PYENV_VERSION=3.7.9 TOXENV=py37 48 | - <<: *linux 49 | env: PYENV_VERSION=3.7-dev TOXENV=py37 50 | - <<: *linux 51 | env: PYENV_VERSION=3.8.7 TOXENV=py38 52 | - <<: *linux 53 | env: PYENV_VERSION=3.8-dev TOXENV=py38 54 | - <<: *linux 55 | env: PYENV_VERSION=3.9.1 TOXENV=py39 56 | - <<: *linux 57 | env: PYENV_VERSION=3.9-dev TOXENV=py39 58 | - <<: *linux 59 | env: PYENV_VERSION=pypy2.7-7.3.1 TOXENV=pypy 60 | - <<: *linux 61 | env: PYENV_VERSION=pypy3.6-7.3.1 TOXENV=pypy3 62 | - &osx 63 | os: osx 64 | osx_image: xcode12.2 65 | language: generic 66 | env: PYENV_VERSION=2.7.18 TOXENV=py27 67 | - <<: *osx 68 | env: PYENV_VERSION=2.7-dev TOXENV=py27 69 | - <<: *osx 70 | env: PYENV_VERSION=3.5.10 TOXENV=py35 71 | - <<: *osx 72 | env: PYENV_VERSION=3.5-dev TOXENV=py35 73 | - <<: *osx 74 | env: PYENV_VERSION=3.6.12 TOXENV=py36 75 | - <<: *osx 76 | env: PYENV_VERSION=3.6-dev TOXENV=py36 77 | - <<: *osx 78 | env: PYENV_VERSION=3.7.9 TOXENV=py37 79 | - <<: *osx 80 | env: PYENV_VERSION=3.7-dev TOXENV=py37 81 | - <<: *osx 82 | env: PYENV_VERSION=3.8.7 TOXENV=py38 83 | - <<: *osx 84 | env: PYENV_VERSION=3.8-dev TOXENV=py38 85 | - <<: *osx 86 | env: PYENV_VERSION=3.9.1 TOXENV=py39 87 | - <<: *osx 88 | env: PYENV_VERSION=3.9-dev TOXENV=py39 89 | - <<: *osx 90 | env: PYENV_VERSION=pypy2.7-7.3.1 TOXENV=pypy 91 | - <<: *osx 92 | env: PYENV_VERSION=pypy3.6-7.3.1 TOXENV=pypy3 93 | allow_failures: 94 | - env: PYENV_VERSION=2.7-dev TOXENV=py27 95 | - env: PYENV_VERSION=3.5-dev TOXENV=py35 96 | - env: PYENV_VERSION=3.6-dev TOXENV=py36 97 | - env: PYENV_VERSION=3.7-dev TOXENV=py37 98 | - env: PYENV_VERSION=3.8-dev TOXENV=py38 99 | - env: PYENV_VERSION=3.9-dev TOXENV=py39 100 | 101 | before_install: 102 | - mkdir -p "${PYTHON_BUILD_CACHE_PATH}" 103 | - if [ "$TRAVIS_OS_NAME" = "linux" ]; then git -C "$(pyenv root)" fetch && git -C "$(pyenv root)" checkout master && git -C "$(pyenv root)" pull; fi 104 | 105 | install: 106 | - python -m pip install -r travis-requirements.txt 107 | - pyenv install $(pyenv exec python --version >/dev/null 2>&1 && echo "--skip-existing" || echo "--force") "${PYENV_VERSION}" 108 | 109 | script: 110 | - python -m tox -vv -- --cov-report=xml 111 | 112 | after_success: 113 | - python -m codecov --required 114 | 115 | before_deploy: 116 | - python setup.py sdist bdist_wheel 117 | 118 | deploy: 119 | - provider: pypi 120 | username: Ilya.Kulakov 121 | password: 122 | secure: "fDYi/HJvYyqUggKmN/Dc6YewUsBAzHWBdTYMpDfHETeOIvv2G268atnIwcoWav63fwPUpagwOlzQhRklqeLRmjEzr4M+wzFsAQVAnj6a7ChLPWPmgZlClFRpm6leWZjzGD+1FPnH/vvwTHlDi7j+1zgfh4WEellnw3hU+Lzjx+o=" 123 | distributions: "sdist bdist_wheel" 124 | on: 125 | tags: true 126 | branch: master 127 | condition: $DEPLOY = 1 128 | - provider: releases 129 | cleanup: false 130 | token: 131 | secure: "QHn7vzWo7rbgemP37qdNU4h+q7Xb2CQ7HxPFfa7yTsxFd8V4+sQLVrnaQtzYTM8dJWvRgi8PVHVGl2VGnQAiRM4Nd/NE/3HL9aHQIfWRtZ6XHfNVQ55bxJzLfZZy2M+32b8W268ELj3ty4C3Mo7TuOTv4svQoRDrLzGozJCpu+w=" 132 | file_glob: true 133 | file: dist/* 134 | on: 135 | tags: true 136 | branch: master 137 | condition: $DEPLOY = 1 138 | -------------------------------------------------------------------------------- /CHANGES.rst: -------------------------------------------------------------------------------- 1 | CHANGES 2 | ======= 3 | 4 | 1.23.1 (2022-10-02) 5 | ------------------- 6 | 7 | - Include typing information in PyPI and source distributions. 8 | 9 | 1.23.0 (2021-01-29) 10 | ------------------- 11 | 12 | - List of submodules is retrieved via git instead of parsing .gitmodules. See #85 13 | 14 | 1.22.0 (2020-08-04) 15 | ------------------- 16 | 17 | - Fixed the --no-exclude flag. See #82 18 | - --no-exclude -> --no-export-ignore (backward compatibility is retained) 19 | - --extra -> --include (backward compatibility is retained) 20 | 21 | 1.21.0 (2020-02-11) 22 | ------------------- 23 | 24 | - Fixed handling of files inside export-ignore directories (#78) 25 | 26 | 1.20.0 (2019-11-07) 27 | ------------------- 28 | 29 | - Fixed handling of non-unicode byte sequences on Linux 30 | - Fixed parsing of git version on Windows 31 | - Added support for path-like objects to GitArchiver 32 | 33 | 1.19.4 (2018-12-07) 34 | ------------------- 35 | 36 | - Fixed compatibility with Apple's git (bundled with Xcode) 37 | 38 | 1.19.3 (2018-11-27) 39 | ------------------- 40 | 41 | - Add the git_version parameter to GitArchiver and the get_git_version class method 42 | - If git version (initialized or guessed) is less than 1.6.1, exception is raised 43 | - Properly read non-nul separated output of check-attr if git version is less than 1.8.5. See #65 44 | 45 | **Known Bugs:** 46 | 47 | - Does not work with Apple's git (bundled with Xcode). See #68 48 | 49 | 1.19.2 (2018-11-13) 50 | ------------------- 51 | 52 | - Support Windows 53 | - Fix missing pycodestyle in setup.py's tests_require 54 | 55 | 1.19.1 (2018-11-01) 56 | ------------------- 57 | 58 | - Fix passing compresslevel=None may cause segfault on some systems 59 | 60 | 1.19.0 (2018-10-31) 61 | ------------------- 62 | 63 | - 🎃 64 | - Use -0 ... -9 to explicitly specify compression level if format allows; if unset, lib's default is used 65 | - Checking for file exclusion is optimized, the process is spawned only once per repo / submodule 66 | 67 | **Known Bugs:** 68 | 69 | - Not passing a compression level explicitly `[-0 | ... | -9]` may cause a segfault. See #59 70 | 71 | 1.18.3 (2018-09-27) 72 | ------------------- 73 | 74 | - Fix broken support for zip files 75 | 76 | 1.18.2 (2018-09-19) 77 | ------------------- 78 | 79 | - Fix redundant print 80 | - Fix mismatch between dry-run and normal verbose logging 81 | - Fix missing support for tbz2 files 82 | - API: Raise ValueError instead of RuntimeError if output format is not recognized 83 | - API: Conditionally import zipfile / tarfile depending on requested output format 84 | 85 | 1.18.1 (2018-09-01) 86 | ------------------- 87 | 88 | - Improve support for special characters 89 | 90 | 1.18.0 (2018-08-14) 91 | ------------------- 92 | 93 | - Add **CHANGES.rst** to track further changes 94 | - Add tests 95 | - Use `git check-attr` to test against export-ignore 96 | - Better support for unicode file names 97 | - Require Git >= 1.6.1 98 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2010 Ilya Kulakov 2 | 3 | 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | 13 | 14 | The above copyright notice and this permission notice shall be included in 15 | all copies or substantial portions of the Software. 16 | 17 | 18 | 19 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 20 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 21 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 22 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 23 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 24 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 25 | THE SOFTWARE. 26 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include *.rst 2 | include *.typed 3 | include *.pyi -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | prefix=/usr/local 2 | SOURCE_FILE=git_archive_all.py 3 | TARGET_DIR=$(prefix)/bin 4 | TARGET_FILE=$(TARGET_DIR)/git-archive-all 5 | 6 | all: 7 | @echo "usage: make install" 8 | @echo " make uninstall" 9 | @echo " test" 10 | 11 | test: 12 | python setup.py test 13 | 14 | install: 15 | install -d -m 0755 $(TARGET_DIR) 16 | install -m 0755 $(SOURCE_FILE) $(TARGET_FILE) 17 | 18 | uninstall: 19 | test -d $(TARGET_DIR) && \ 20 | rm -f $(TARGET_FILE) 21 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | | |pypi| |homebrew| 2 | | |implementations| |versions| 3 | | |travis| |coverage| 4 | 5 | .. |pypi| image:: https://img.shields.io/pypi/v/git-archive-all.svg 6 | :target: https://pypi.python.org/pypi/git-archive-all 7 | :alt: PyPI 8 | .. |homebrew| image:: https://img.shields.io/homebrew/v/git-archive-all.svg 9 | :target: https://formulae.brew.sh/formula/git-archive-all 10 | :alt: Homebrew 11 | .. |versions| image:: https://img.shields.io/pypi/pyversions/git-archive-all.svg 12 | :target: https://pypi.python.org/pypi/git-archive-all 13 | :alt: Supported Python versions 14 | .. |implementations| image:: https://img.shields.io/pypi/implementation/git-archive-all.svg 15 | :target: https://pypi.python.org/pypi/git-archive-all 16 | :alt: Supported Python implementations 17 | .. |travis| image:: https://travis-ci.org/Kentzo/git-archive-all.svg?branch=master 18 | :target: https://travis-ci.org/Kentzo/git-archive-all 19 | :alt: Travis 20 | .. |coverage| image:: https://codecov.io/gh/Kentzo/git-archive-all/branch/master/graph/badge.svg 21 | :target: https://codecov.io/gh/Kentzo/git-archive-all/branch/master 22 | :alt: Coverage 23 | 24 | Archive a repository with all its submodules. 25 | 26 | :: 27 | 28 | git-archive-all [-v] [-C BASE_REPO] [--prefix PREFIX] [--no-export-ignore] [--force-submodules] [--include EXTRA1 ...] [--dry-run] [-0 | ... | -9] OUTPUT_FILE 29 | 30 | Options: 31 | 32 | --version show program's version number and exit 33 | 34 | -h, --help show this help message and exit 35 | 36 | -v, --verbose enable verbose mode 37 | 38 | --prefix=PREFIX prepend PREFIX to each filename in the archive; 39 | defaults to OUTPUT_FILE name 40 | 41 | -C BASE_REPO use BASE_REPO as the main git repository to archive; 42 | defaults to the current directory when empty 43 | 44 | --no-export-ignore ignore the [-]export-ignore attribute in .gitattributes 45 | 46 | --force-submodules force `git submodule init && git submodule update` at 47 | each level before iterating submodules 48 | 49 | --include=EXTRA additional files to include in the archive 50 | 51 | --dry-run show files to be archived without actually creating the archive 52 | 53 | Questions & Answers 54 | ------------------- 55 | 56 | | Q: How to exclude files? 57 | | A: Mark paths you want to exclude in the .gitattributes file with the export-ignore attribute. Read more on `git-scm.com `_. 58 | 59 | | Q: What about non-unicode filenames? 60 | | A: All filenames that particular version of Python can represent and handle are supported. Extra [en|de]coding is done where appropriate. 61 | 62 | Support 63 | ------- 64 | If functional you need is missing but you're ready to pay for it, feel free to `contact me `_. If not, create an issue anyway, I'll take a look as soon as I can. 65 | -------------------------------------------------------------------------------- /appveyor-requirements.txt: -------------------------------------------------------------------------------- 1 | tox==3.20.1 2 | codecov==2.1.10 3 | tox-venv==0.4.0 4 | -------------------------------------------------------------------------------- /git_archive_all.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # coding=utf-8 3 | 4 | # The MIT License (MIT) 5 | # 6 | # Copyright (c) 2010 Ilya Kulakov 7 | # 8 | # Permission is hereby granted, free of charge, to any person obtaining a copy 9 | # of this software and associated documentation files (the "Software"), to deal 10 | # in the Software without restriction, including without limitation the rights 11 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 12 | # copies of the Software, and to permit persons to whom the Software is 13 | # furnished to do so, subject to the following conditions: 14 | # 15 | # The above copyright notice and this permission notice shall be included in 16 | # all copies or substantial portions of the Software. 17 | # 18 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 21 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 22 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 23 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 24 | # THE SOFTWARE. 25 | 26 | from __future__ import print_function 27 | from __future__ import unicode_literals 28 | 29 | import logging 30 | from os import environ, extsep, path, readlink 31 | from subprocess import CalledProcessError, Popen, PIPE 32 | import sys 33 | import re 34 | 35 | __version__ = "1.23.1" 36 | 37 | 38 | try: 39 | # Python 3.2+ 40 | from os import fsdecode 41 | except ImportError: 42 | def fsdecode(filename): 43 | if not isinstance(filename, unicode): 44 | return filename.decode(sys.getfilesystemencoding(), 'strict') 45 | else: 46 | return filename 47 | 48 | try: 49 | # Python 3.2+ 50 | from os import fsencode 51 | except ImportError: 52 | def fsencode(filename): 53 | if not isinstance(filename, bytes): 54 | return filename.encode(sys.getfilesystemencoding(), 'strict') 55 | else: 56 | return filename 57 | 58 | 59 | def git_fsdecode(filename): 60 | """ 61 | Decode filename from git output into str. 62 | """ 63 | if sys.platform.startswith('win32'): 64 | return filename.decode('utf-8') 65 | else: 66 | return fsdecode(filename) 67 | 68 | 69 | def git_fsencode(filename): 70 | """ 71 | Encode filename from str into git input. 72 | """ 73 | if sys.platform.startswith('win32'): 74 | return filename.encode('utf-8') 75 | else: 76 | return fsencode(filename) 77 | 78 | 79 | try: 80 | # Python 3.6+ 81 | from os import fspath as _fspath 82 | 83 | def fspath(filename, decoder=fsdecode, encoder=fsencode): 84 | """ 85 | Convert filename into bytes or str, depending on what's the best type 86 | to represent paths for current Python and platform. 87 | """ 88 | # Python 3.6+: str can represent any path (PEP 383) 89 | # str is not required on Windows (PEP 529) 90 | # Decoding is still applied for consistency and to follow PEP 519 recommendation. 91 | return decoder(_fspath(filename)) 92 | except ImportError: 93 | def fspath(filename, decoder=fsdecode, encoder=fsencode): 94 | # Python 3.4 and 3.5: str can represent any path (PEP 383), 95 | # but str is required on Windows (no PEP 529) 96 | # 97 | # Python 2.6 and 2.7: str cannot represent any path (no PEP 383), 98 | # str is required on Windows (no PEP 529) 99 | # bytes is required on POSIX (no PEP 383) 100 | if sys.version_info > (3,): 101 | import pathlib 102 | if isinstance(filename, pathlib.PurePath): 103 | return str(filename) 104 | else: 105 | return decoder(filename) 106 | elif sys.platform.startswith('win32'): 107 | return decoder(filename) 108 | else: 109 | return encoder(filename) 110 | 111 | 112 | def git_fspath(filename): 113 | """ 114 | fspath representation of git output. 115 | """ 116 | return fspath(filename, git_fsdecode, git_fsencode) 117 | 118 | 119 | class GitArchiver(object): 120 | """ 121 | GitArchiver 122 | 123 | Scan a git repository and export all tracked files, and submodules. 124 | Checks for .gitattributes files in each directory and uses 'export-ignore' 125 | pattern entries for ignore files in the archive. 126 | 127 | >>> archiver = GitArchiver(main_repo_abspath='my/repo/path') 128 | >>> archiver.create('output.zip') 129 | """ 130 | TARFILE_FORMATS = { 131 | 'tar': 'w', 132 | 'tbz2': 'w:bz2', 133 | 'tgz': 'w:gz', 134 | 'txz': 'w:xz', 135 | 'bz2': 'w:bz2', 136 | 'gz': 'w:gz', 137 | 'xz': 'w:xz' 138 | } 139 | ZIPFILE_FORMATS = ('zip',) 140 | 141 | LOG = logging.getLogger('GitArchiver') 142 | 143 | def __init__(self, prefix='', exclude=True, force_sub=False, extra=None, main_repo_abspath=None, git_version=None): 144 | """ 145 | @param prefix: Prefix used to prepend all paths in the resulting archive. 146 | Extra file paths are only prefixed if they are not relative. 147 | E.g. if prefix is 'foo' and extra is ['bar', '/baz'] the resulting archive will look like this: 148 | / 149 | baz 150 | foo/ 151 | bar 152 | 153 | @param exclude: Determines whether archiver should follow rules specified in .gitattributes files. 154 | 155 | @param force_sub: Determines whether submodules are initialized and updated before archiving. 156 | 157 | @param extra: List of extra paths to include in the resulting archive. 158 | 159 | @param main_repo_abspath: Absolute path to the main repository (or one of subdirectories). 160 | If given path is path to a subdirectory (but not a submodule directory!) it will be replaced 161 | with abspath to top-level directory of the repository. 162 | If None, current cwd is used. 163 | 164 | @param git_version: Version of Git that determines whether various workarounds are on. 165 | If None, tries to resolve via Git's CLI. 166 | """ 167 | self._check_attr_gens = {} 168 | self._ignored_paths_cache = {} 169 | 170 | if git_version is None: 171 | git_version = self.get_git_version() 172 | 173 | if git_version is not None and git_version < (1, 6, 1): 174 | raise ValueError("git of version 1.6.1 and higher is required") 175 | 176 | self.git_version = git_version 177 | 178 | if main_repo_abspath is None: 179 | main_repo_abspath = path.abspath('') 180 | elif not path.isabs(main_repo_abspath): 181 | raise ValueError("main_repo_abspath must be an absolute path") 182 | 183 | self.main_repo_abspath = self.resolve_git_main_repo_abspath(main_repo_abspath) 184 | 185 | self.prefix = fspath(prefix) 186 | self.exclude = exclude 187 | self.extra = [fspath(e) for e in extra] if extra is not None else [] 188 | self.force_sub = force_sub 189 | 190 | def create(self, output_path, dry_run=False, output_format=None, compresslevel=None): 191 | """ 192 | Create the archive at output_file_path. 193 | 194 | Type of the archive is determined either by extension of output_file_path or by output_format. 195 | Supported formats are: gz, zip, bz2, xz, tar, tgz, txz 196 | 197 | @param output_path: Output file path. 198 | 199 | @param dry_run: Determines whether create should do nothing but print what it would archive. 200 | 201 | @param output_format: Determines format of the output archive. If None, format is determined from extension 202 | of output_file_path. 203 | 204 | @param compresslevel: Optional compression level. Interpretation depends on the output format. 205 | """ 206 | output_path = fspath(output_path) 207 | 208 | if output_format is None: 209 | file_name, file_ext = path.splitext(output_path) 210 | output_format = file_ext[len(extsep):].lower() 211 | self.LOG.debug("Output format is not explicitly set, determined format is {0}.".format(output_format)) 212 | 213 | if not dry_run: 214 | if output_format in self.ZIPFILE_FORMATS: 215 | from zipfile import ZipFile, ZipInfo, ZIP_DEFLATED 216 | 217 | if compresslevel is not None: 218 | if sys.version_info > (3, 7): 219 | archive = ZipFile(path.abspath(output_path), 'w', compresslevel=compresslevel) 220 | else: 221 | raise ValueError("Compression level for zip archives requires Python 3.7+") 222 | else: 223 | archive = ZipFile(path.abspath(output_path), 'w') 224 | 225 | def add_file(file_path, arcname): 226 | if not path.islink(file_path): 227 | archive.write(file_path, arcname, ZIP_DEFLATED) 228 | else: 229 | i = ZipInfo(arcname) 230 | i.create_system = 3 231 | i.external_attr = 0xA1ED0000 232 | archive.writestr(i, readlink(file_path)) 233 | elif output_format in self.TARFILE_FORMATS: 234 | import tarfile 235 | 236 | mode = self.TARFILE_FORMATS[output_format] 237 | 238 | if compresslevel is not None: 239 | try: 240 | archive = tarfile.open(path.abspath(output_path), mode, compresslevel=compresslevel) 241 | except TypeError: 242 | raise ValueError("{0} cannot be compressed".format(output_format)) 243 | else: 244 | archive = tarfile.open(path.abspath(output_path), mode) 245 | 246 | def add_file(file_path, arcname): 247 | archive.add(file_path, arcname) 248 | else: 249 | raise ValueError("unknown format: {0}".format(output_format)) 250 | 251 | def archiver(file_path, arcname): 252 | self.LOG.debug(fspath("{0} => {1}").format(file_path, arcname)) 253 | add_file(file_path, arcname) 254 | else: 255 | archive = None 256 | 257 | def archiver(file_path, arcname): 258 | self.LOG.info(fspath("{0} => {1}").format(file_path, arcname)) 259 | 260 | self.archive_all_files(archiver) 261 | 262 | if archive is not None: 263 | archive.close() 264 | 265 | def is_file_excluded(self, repo_abspath, repo_file_path): 266 | """ 267 | Checks whether file at a given path is excluded. 268 | 269 | @param repo_abspath: Absolute path to the git repository. 270 | 271 | @param repo_file_path: Path to a file relative to repo_abspath. 272 | 273 | @return: True if file should be excluded. Otherwise False. 274 | """ 275 | if not self.exclude: 276 | return False 277 | 278 | cache = self._ignored_paths_cache.setdefault(repo_abspath, {}) 279 | 280 | if repo_file_path not in cache: 281 | next(self._check_attr_gens[repo_abspath]) 282 | attrs = self._check_attr_gens[repo_abspath].send(repo_file_path) 283 | export_ignore_attr = attrs['export-ignore'] 284 | 285 | if export_ignore_attr == b'set': 286 | cache[repo_file_path] = True 287 | elif export_ignore_attr == b'unset': 288 | cache[repo_file_path] = False 289 | else: 290 | repo_file_dir_path = path.dirname(repo_file_path) 291 | 292 | if repo_file_dir_path: 293 | cache[repo_file_path] = self.is_file_excluded(repo_abspath, repo_file_dir_path) 294 | else: 295 | cache[repo_file_path] = False 296 | 297 | return cache[repo_file_path] 298 | 299 | def archive_all_files(self, archiver): 300 | """ 301 | Archive all files using archiver. 302 | 303 | @param archiver: Callable that accepts 2 arguments: 304 | abspath to file on the system and relative path within archive. 305 | """ 306 | for file_path in self.extra: 307 | archiver(path.abspath(file_path), path.join(self.prefix, file_path)) 308 | 309 | for file_path in self.walk_git_files(): 310 | archiver(path.join(self.main_repo_abspath, file_path), path.join(self.prefix, file_path)) 311 | 312 | def walk_git_files(self, repo_path=fspath('')): 313 | """ 314 | An iterator method that yields a file path relative to main_repo_abspath 315 | for each file that should be included in the archive. 316 | Skips those that match the exclusion patterns found in 317 | any discovered .gitattributes files along the way. 318 | 319 | Recurs into submodules as well. 320 | 321 | @param repo_path: Path to the git submodule repository relative to main_repo_abspath. 322 | 323 | @return: Generator to traverse files under git control relative to main_repo_abspath. 324 | """ 325 | repo_abspath = path.join(self.main_repo_abspath, fspath(repo_path)) 326 | assert repo_abspath not in self._check_attr_gens 327 | self._check_attr_gens[repo_abspath] = self.check_git_attr(repo_abspath, ['export-ignore']) 328 | 329 | try: 330 | repo_file_paths = self.list_repo_files(repo_abspath) 331 | 332 | for repo_file_path in repo_file_paths: 333 | repo_file_abspath = path.join(repo_abspath, repo_file_path) # absolute file path 334 | main_repo_file_path = path.join(repo_path, repo_file_path) # relative to main_repo_abspath 335 | 336 | if not path.islink(repo_file_abspath) and path.isdir(repo_file_abspath): 337 | continue 338 | 339 | if self.is_file_excluded(repo_abspath, repo_file_path): 340 | continue 341 | 342 | yield main_repo_file_path 343 | 344 | if self.force_sub: 345 | self.run_git_shell('git submodule init', repo_abspath) 346 | self.run_git_shell('git submodule update', repo_abspath) 347 | 348 | for repo_submodule_path in self.list_repo_submodules(repo_abspath): # relative to repo_path 349 | if self.is_file_excluded(repo_abspath, repo_submodule_path): 350 | continue 351 | 352 | main_repo_submodule_path = path.join(repo_path, repo_submodule_path) # relative to main_repo_abspath 353 | 354 | for main_repo_submodule_file_path in self.walk_git_files(main_repo_submodule_path): 355 | repo_submodule_file_path = path.relpath(main_repo_submodule_file_path, repo_path) # relative to repo_path 356 | 357 | if self.is_file_excluded(repo_abspath, repo_submodule_file_path): 358 | continue 359 | 360 | yield main_repo_submodule_file_path 361 | finally: 362 | self._check_attr_gens[repo_abspath].close() 363 | del self._check_attr_gens[repo_abspath] 364 | 365 | def check_git_attr(self, repo_abspath, attrs): 366 | """ 367 | Generator that returns git attributes for received paths relative to repo_abspath. 368 | 369 | >>> archiver = GitArchiver(...) 370 | >>> g = archiver.check_git_attr('repo_path', ['export-ignore']) 371 | >>> next(g) 372 | >>> attrs = g.send('relative_path') 373 | >>> print(attrs['export-ignore']) 374 | 375 | @param repo_abspath: Absolute path to a git repository. 376 | 377 | @param attrs: Attributes to check 378 | """ 379 | def make_process(): 380 | env = dict(environ, GIT_FLUSH='1') 381 | cmd = 'git check-attr --stdin -z {0}'.format(' '.join(attrs)) 382 | return Popen(cmd, shell=True, stdin=PIPE, stdout=PIPE, cwd=repo_abspath, env=env) 383 | 384 | def read_attrs(process, repo_file_path): 385 | process.stdin.write(repo_file_path + b'\0') 386 | process.stdin.flush() 387 | 388 | # For every attribute check-attr will output: NUL NUL NUL 389 | path, attr, info = b'', b'', b'' 390 | nuls_count = 0 391 | nuls_expected = 3 * len(attrs) 392 | 393 | while nuls_count != nuls_expected: 394 | b = process.stdout.read(1) 395 | 396 | if b == b'' and process.poll() is not None: 397 | raise RuntimeError("check-attr exited prematurely") 398 | elif b == b'\0': 399 | nuls_count += 1 400 | 401 | if nuls_count % 3 == 0: 402 | yield path, attr, info 403 | 404 | path, attr, info = b'', b'', b'' 405 | elif nuls_count % 3 == 0: 406 | path += b 407 | elif nuls_count % 3 == 1: 408 | attr += b 409 | elif nuls_count % 3 == 2: 410 | info += b 411 | 412 | def read_attrs_old(process, repo_file_path): 413 | """ 414 | Compatibility with versions 1.8.5 and below that do not recognize -z for output. 415 | """ 416 | process.stdin.write(repo_file_path + b'\0') 417 | process.stdin.flush() 418 | 419 | # For every attribute check-attr will output: : : \n 420 | # where is c-quoted 421 | 422 | path, attr, info = b'', b'', b'' 423 | lines_count = 0 424 | lines_expected = len(attrs) 425 | 426 | while lines_count != lines_expected: 427 | line = process.stdout.readline() 428 | 429 | info_start = line.rfind(b': ') 430 | if info_start == -1: 431 | raise RuntimeError("unexpected output of check-attr: {0}".format(line)) 432 | 433 | attr_start = line.rfind(b': ', 0, info_start) 434 | if attr_start == -1: 435 | raise RuntimeError("unexpected output of check-attr: {0}".format(line)) 436 | 437 | path = line[:attr_start] 438 | attr = line[attr_start + 2:info_start] # trim leading ": " 439 | info = line[info_start + 2:len(line) - 1] # trim leading ": " and trailing \n 440 | yield path, attr, info 441 | 442 | lines_count += 1 443 | 444 | if not attrs: 445 | return 446 | 447 | process = make_process() 448 | 449 | if self.git_version is None or self.git_version > (1, 8, 5): 450 | reader = read_attrs 451 | else: 452 | reader = read_attrs_old 453 | 454 | try: 455 | while True: 456 | repo_file_path = yield 457 | repo_file_path = git_fsencode(fspath(repo_file_path)) 458 | repo_file_attrs = {} 459 | 460 | for path, attr, value in reader(process, repo_file_path): 461 | attr = attr.decode('utf-8') 462 | repo_file_attrs[attr] = value 463 | 464 | yield repo_file_attrs 465 | finally: 466 | process.stdin.close() 467 | process.wait() 468 | 469 | def resolve_git_main_repo_abspath(self, abspath): 470 | """ 471 | Return absolute path to the repo for a given path. 472 | """ 473 | try: 474 | main_repo_abspath = self.run_git_shell('git rev-parse --show-toplevel', cwd=abspath).rstrip() 475 | return path.abspath(git_fspath(main_repo_abspath)) 476 | except CalledProcessError as e: 477 | raise ValueError("{0} is not part of a git repository ({1})".format(abspath, e.returncode)) 478 | 479 | @classmethod 480 | def run_git_shell(cls, cmd, cwd=None): 481 | """ 482 | Run git shell command, read output and decode it into a unicode string. 483 | 484 | @param cmd: Command to be executed. 485 | 486 | @param cwd: Working directory. 487 | 488 | @return: Output of the command. 489 | 490 | @raise CalledProcessError: Raises exception if return code of the command is non-zero. 491 | """ 492 | p = Popen(cmd, shell=True, stdout=PIPE, cwd=cwd) 493 | output, _ = p.communicate() 494 | 495 | if p.returncode: 496 | if sys.version_info > (2, 6): 497 | raise CalledProcessError(returncode=p.returncode, cmd=cmd, output=output) 498 | else: 499 | raise CalledProcessError(returncode=p.returncode, cmd=cmd) 500 | 501 | return output 502 | 503 | @classmethod 504 | def get_git_version(cls): 505 | """ 506 | Return version of git current shell points to. 507 | 508 | If version cannot be parsed None is returned. 509 | """ 510 | try: 511 | output = cls.run_git_shell('git version') 512 | except CalledProcessError: 513 | cls.LOG.warning("Unable to get Git version.") 514 | return None 515 | 516 | try: 517 | version = output.split()[2] 518 | except IndexError: 519 | cls.LOG.warning("Unable to parse Git version \"%s\".", output) 520 | return None 521 | 522 | try: 523 | return tuple(int(v) if v.isdigit() else 0 for v in version.split(b'.')) 524 | except ValueError: 525 | cls.LOG.warning("Unable to parse Git version \"%s\".", version) 526 | return None 527 | 528 | @classmethod 529 | def list_repo_files(cls, repo_abspath): 530 | """ 531 | Return a list of all files as seen by git in a given repo. 532 | """ 533 | repo_file_paths = cls.run_git_shell( 534 | 'git ls-files -z --cached --full-name --no-empty-directory', 535 | cwd=repo_abspath 536 | ) 537 | repo_file_paths = repo_file_paths.split(b'\0')[:-1] 538 | 539 | if sys.platform.startswith('win32'): 540 | repo_file_paths = (git_fspath(p.replace(b'/', b'\\')) for p in repo_file_paths) 541 | else: 542 | repo_file_paths = map(git_fspath, repo_file_paths) 543 | 544 | return repo_file_paths 545 | 546 | @classmethod 547 | def list_repo_submodules(cls, repo_abspath): 548 | """ 549 | Return a list of all direct submodules as seen by git in a given repo. 550 | """ 551 | if sys.platform.startswith('win32'): 552 | shell_command = 'git submodule foreach --quiet "\\"{0}\\" -c \\"from __future__ import print_function; print(\'"$sm_path"\', end=chr(0))\\""' 553 | else: 554 | shell_command = 'git submodule foreach --quiet \'"{0}" -c "from __future__ import print_function; print(\\"$sm_path\\", end=chr(0))"\'' 555 | 556 | python_exe = sys.executable or 'python' 557 | shell_command = shell_command.format(python_exe) 558 | 559 | repo_submodule_paths = cls.run_git_shell(shell_command, cwd=repo_abspath) 560 | repo_submodule_paths = repo_submodule_paths.split(b'\0')[:-1] 561 | 562 | if sys.platform.startswith('win32'): 563 | repo_submodule_paths = (git_fspath(p.replace(b'/', b'\\')) for p in repo_submodule_paths) 564 | else: 565 | repo_submodule_paths = map(git_fspath, repo_submodule_paths) 566 | 567 | return repo_submodule_paths 568 | 569 | 570 | def main(argv=None): 571 | if argv is None: 572 | argv = sys.argv 573 | 574 | from optparse import OptionParser, SUPPRESS_HELP 575 | 576 | parser = OptionParser( 577 | usage="usage: %prog [-v] [-C BASE_REPO] [--prefix PREFIX] [--no-export-ignore]" 578 | " [--force-submodules] [--include EXTRA1 ...] [--dry-run] [-0 | ... | -9] OUTPUT_FILE", 579 | version="%prog {0}".format(__version__) 580 | ) 581 | 582 | parser.add_option('--prefix', 583 | type='string', 584 | dest='prefix', 585 | default=None, 586 | help="""prepend PREFIX to each filename in the archive; 587 | defaults to OUTPUT_FILE name""") 588 | 589 | parser.add_option('-C', 590 | type='string', 591 | dest='base_repo', 592 | default=None, 593 | help="""use BASE_REPO as the main git repository to archive; 594 | defaults to the current directory when empty""") 595 | 596 | parser.add_option('-v', '--verbose', 597 | action='store_true', 598 | dest='verbose', 599 | help='enable verbose mode') 600 | 601 | parser.add_option('--no-export-ignore', '--no-exclude', 602 | action='store_false', 603 | dest='exclude', 604 | default=True, 605 | help="ignore the [-]export-ignore attribute in .gitattributes") 606 | 607 | parser.add_option('--force-submodules', 608 | action='store_true', 609 | dest='force_sub', 610 | help='force `git submodule init && git submodule update` at each level before iterating submodules') 611 | 612 | parser.add_option('--include', '--extra', 613 | action='append', 614 | dest='extra', 615 | default=[], 616 | help="additional files to include in the archive") 617 | 618 | parser.add_option('--dry-run', 619 | action='store_true', 620 | dest='dry_run', 621 | help="show files to be archived without actually creating the archive") 622 | 623 | for i in range(10): 624 | parser.add_option('-{0}'.format(i), 625 | action='store_const', 626 | const=i, 627 | dest='compresslevel', 628 | help=SUPPRESS_HELP) 629 | 630 | options, args = parser.parse_args(argv[1:]) 631 | 632 | if len(args) != 1: 633 | parser.error("You must specify exactly one output file") 634 | 635 | output_file_path = args[0] 636 | 637 | if path.isdir(output_file_path): 638 | parser.error("You cannot use directory as output") 639 | 640 | # avoid tarbomb 641 | if options.prefix is not None: 642 | options.prefix = path.join(options.prefix, '') 643 | else: 644 | output_name = path.basename(output_file_path) 645 | output_name = re.sub( 646 | '(\\.zip|\\.tar|\\.tbz2|\\.tgz|\\.txz|\\.bz2|\\.gz|\\.xz|\\.tar\\.bz2|\\.tar\\.gz|\\.tar\\.xz)$', 647 | '', 648 | output_name 649 | ) or "Archive" 650 | options.prefix = path.join(output_name, '') 651 | 652 | try: 653 | handler = logging.StreamHandler(sys.stdout) 654 | handler.setFormatter(logging.Formatter('%(message)s')) 655 | GitArchiver.LOG.addHandler(handler) 656 | GitArchiver.LOG.setLevel(logging.DEBUG if options.verbose else logging.INFO) 657 | archiver = GitArchiver(options.prefix, 658 | options.exclude, 659 | options.force_sub, 660 | options.extra, 661 | path.abspath(options.base_repo) if options.base_repo is not None else None 662 | ) 663 | archiver.create(output_file_path, options.dry_run, compresslevel=options.compresslevel) 664 | except Exception as e: 665 | parser.exit(2, "{0}\n".format(e)) 666 | 667 | return 0 668 | 669 | 670 | if __name__ == '__main__': 671 | sys.exit(main()) 672 | -------------------------------------------------------------------------------- /git_archive_all.pyi: -------------------------------------------------------------------------------- 1 | from os import PathLike as _PathLike 2 | import logging 3 | from typing import Callable, Collection, ClassVar, Dict, Generator, Iterable, List, Optional, Tuple, Union 4 | 5 | PathLike = Union[str, bytes, _PathLike] 6 | PathStr = Union[str, bytes] 7 | CheckGitAttrGen = Generator[Dict[str, bytes], PathStr, None] 8 | 9 | def fsdecode(filename: PathLike) -> str: ... 10 | 11 | def fsencode(filename: PathLike) -> bytes: ... 12 | 13 | def git_fsdecode(filename: bytes) -> str: ... 14 | 15 | def git_fsencode(filename: str) -> bytes: ... 16 | 17 | def fspath(filename: PathLike, decoder=Callable[[PathLike], str], encoder=Callable[[PathLike], bytes]) -> PathStr: ... 18 | 19 | def git_fspath(filename: bytes) -> PathStr: ... 20 | 21 | class GitArchiver(object): 22 | TARFILE_FORMATS: ClassVar[Dict[str, str]] 23 | ZIPFILE_FORMATS: ClassVar[Tuple[str]] 24 | LOG: ClassVar[logging.Logger] 25 | 26 | _check_attr_gens: Dict[str, CheckGitAttrGen] 27 | _ignored_paths_cache: Dict[PathStr, Dict[PathStr, bool]] 28 | 29 | git_version: Optional[Tuple[int]] 30 | main_repo_abspath: PathStr 31 | prefix: PathStr 32 | exclude: bool 33 | extra: List[PathStr] 34 | force_sub: bool 35 | 36 | def __init__(self, 37 | prefix: PathLike, 38 | exclude: bool, 39 | force_sub: bool, 40 | extra: Iterable[PathLike] = None, 41 | main_repo_abspath: PathLike = None, 42 | git_version: Tuple[int] = None) -> None: ... 43 | 44 | def create(self, 45 | output_path: PathLike, 46 | dry_run: bool, 47 | output_format: str = None, 48 | compresslevel: int = None) -> None: ... 49 | 50 | def is_file_excluded(self, repo_abspath: PathStr, repo_file_path: PathStr) -> bool: ... 51 | 52 | def archive_all_files(self, archiver: Callable[[PathStr, PathStr], None]) -> None: ... 53 | 54 | def walk_git_files(self, repo_path: PathStr = None) -> Generator[PathStr, None, None]: ... 55 | 56 | def check_git_attr(self, repo_abspath: PathStr, attrs: Collection[str]) -> CheckGitAttrGen: ... 57 | 58 | def resolve_git_main_repo_abspath(self, abspath: PathLike) -> PathStr: ... 59 | 60 | @classmethod 61 | def run_git_shell(cls, cmd: str, cwd: PathStr = None) -> bytes: ... 62 | 63 | @classmethod 64 | def get_git_version(cls) -> Optional[Tuple[int]]: ... 65 | 66 | @classmethod 67 | def list_repo_files(cls, repo_abspath: PathStr) -> Generator[PathStr, None, None]: ... 68 | 69 | @classmethod 70 | def list_repo_submodules(cls, repo_abspath: PathStr) -> Generator[PathStr, None, None]: ... 71 | -------------------------------------------------------------------------------- /py.typed: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kentzo/git-archive-all/0e3a2401aa176e5c5be05d66983e31f3b169a524/py.typed -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [bdist_wheel] 2 | universal = 1 3 | 4 | [metadata] 5 | name = git-archive-all 6 | author = Ilya Kulakov 7 | author_email = kulakov.ilya@gmail.com 8 | url = https://github.com/Kentzo/git-archive-all 9 | description = Archive git repository with its submodules. 10 | license = MIT License 11 | license_file = LICENSE.txt 12 | classifiers = 13 | Development Status :: 5 - Production/Stable 14 | Environment :: Console 15 | Intended Audience :: Developers 16 | Intended Audience :: System Administrators 17 | License :: OSI Approved :: MIT License 18 | Natural Language :: English 19 | Programming Language :: Python :: Implementation :: CPython 20 | Programming Language :: Python :: Implementation :: PyPy 21 | Programming Language :: Python :: 2 22 | Programming Language :: Python :: 2.6 23 | Programming Language :: Python :: 2.7 24 | Programming Language :: Python :: 3 25 | Programming Language :: Python :: 3.4 26 | Programming Language :: Python :: 3.5 27 | Programming Language :: Python :: 3.6 28 | Programming Language :: Python :: 3.7 29 | Programming Language :: Python :: 3.8 30 | Programming Language :: Python :: 3.9 31 | Topic :: Software Development :: Version Control 32 | Topic :: System :: Archiving 33 | platforms = 34 | Darwin 35 | Linux 36 | Windows 37 | long_description = file: README.rst 38 | 39 | [options] 40 | zip_safe = 1 41 | python_requires = >=2.6,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*, 42 | tests_require = 43 | colorama==0.4.1; python_version >= "2.7" and python_version < "3.5" 44 | coverage==4.5.4; python_version < "3.5" 45 | importlib-metadata==0.23; python_version >= "2.7" and python_version < "3.5" 46 | more-itertools==7.2.0; python_version >= "3.4" and python_version < "3.5" 47 | pycodestyle==2.4.0; python_version < "2.7" 48 | pycodestyle==2.5.0; python_version >= "2.7" 49 | pytest==3.2.5; python_version < "2.7" 50 | pytest==4.6.6; python_version >= "2.7" and python_version < "3.5" 51 | pytest==5.2.2; python_version >= "3.5" 52 | pytest-cov==2.5.1; python_version < "2.7" 53 | pytest-cov==2.8.1; python_version >= "2.7" 54 | pytest-mock==1.6.3; python_version < "2.7" 55 | pytest-mock==1.11.2; python_version >= "2.7" 56 | setuptools==36.8.0; python_version < "2.7" 57 | zipp==1.1.0; python_version >= "2.7" and python_version < "3.6" 58 | 59 | [tool:pytest] 60 | addopts = --cov=git_archive_all --cov-report=term --cov-branch --showlocals -vv 61 | filterwarnings = error 62 | 63 | [tox:tox] 64 | envlist = 65 | py26 66 | py27 67 | py34 68 | py35 69 | py36 70 | py37 71 | py38 72 | py39 73 | pypy 74 | pypy3 75 | skipsdist = true 76 | 77 | [testenv] 78 | deps = -rtest-requirements.txt 79 | commands = pytest {posargs} 80 | tox_pyenv_fallback = false 81 | 82 | [testenv:py26] 83 | install_command = pip install {opts} {packages} 84 | list_dependencies_command = pip freeze 85 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import re 2 | import sys 3 | 4 | from setuptools import setup 5 | from setuptools.command.test import test as TestCommand 6 | 7 | # Parse the version from the file. 8 | verstrline = open('git_archive_all.py', "rt").read() 9 | VSRE = r"^__version__ = ['\"]([^'\"]*)['\"]" 10 | mo = re.search(VSRE, verstrline, re.M) 11 | if mo: 12 | verstr = mo.group(1) 13 | else: 14 | raise RuntimeError("Unable to find version string in git_archive_all.py") 15 | 16 | 17 | class PyTest(TestCommand): 18 | user_options = [("pytest-args=", "a", "Arguments to pass to pytest")] 19 | 20 | def initialize_options(self): 21 | TestCommand.initialize_options(self) 22 | self.pytest_args = "" 23 | 24 | def run_tests(self): 25 | import shlex 26 | 27 | # import here, cause outside the eggs aren't loaded 28 | import pytest 29 | 30 | errno = pytest.main(shlex.split(self.pytest_args)) 31 | sys.exit(errno) 32 | 33 | 34 | setup( 35 | version=verstr, 36 | py_modules=['git_archive_all'], 37 | package_data={'git_archive_all': ['py.typed', '*.pyi']}, 38 | entry_points={'console_scripts': 'git-archive-all=git_archive_all:main'}, 39 | cmdclass={"test": PyTest}, 40 | ) 41 | -------------------------------------------------------------------------------- /test-requirements.txt: -------------------------------------------------------------------------------- 1 | colorama==0.4.1; python_version >= "2.7" and python_version < "3.5" 2 | coverage==4.5.4; python_version < "3.5" 3 | importlib-metadata==0.23; python_version >= "2.7" and python_version < "3.5" 4 | more-itertools==7.2.0; python_version >= "3.4" and python_version < "3.5" 5 | pycodestyle==2.4.0; python_version < "2.7" 6 | pycodestyle==2.5.0; python_version >= "2.7" 7 | pytest==3.2.5; python_version < "2.7" 8 | pytest==4.6.6; python_version >= "2.7" and python_version < "3.5" 9 | pytest==5.2.2; python_version >= "3.5" 10 | pytest-cov==2.5.1; python_version < "2.7" 11 | pytest-cov==2.8.1; python_version >= "2.7" 12 | pytest-mock==1.6.3; python_version < "2.7" 13 | pytest-mock==1.11.2; python_version >= "2.7" 14 | setuptools==36.8.0; python_version < "2.7" 15 | zipp==1.1.0; python_version >= "2.7" and python_version < "3.6" 16 | -------------------------------------------------------------------------------- /test_git_archive_all.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from __future__ import print_function 4 | from __future__ import unicode_literals 5 | 6 | from copy import deepcopy 7 | import errno 8 | from functools import partial 9 | import os 10 | from subprocess import check_call 11 | import sys 12 | from tarfile import TarFile, PAX_FORMAT 13 | import warnings 14 | 15 | import pycodestyle 16 | import pytest 17 | 18 | import git_archive_all 19 | from git_archive_all import GitArchiver, fspath 20 | 21 | 22 | def makedirs(p): 23 | try: 24 | os.makedirs(p) 25 | except OSError as e: 26 | if e.errno != errno.EEXIST: 27 | raise 28 | 29 | 30 | def as_posix(p): 31 | if sys.platform.startswith('win32'): 32 | return p.replace(b'\\', b'/') if isinstance(p, bytes) else p.replace('\\', '/') 33 | else: 34 | return p 35 | 36 | 37 | def os_path_join(*args): 38 | """ 39 | Ensure that all path components are uniformly encoded. 40 | """ 41 | return os.path.join(*(fspath(p) for p in args)) 42 | 43 | 44 | @pytest.fixture 45 | def git_env(tmpdir_factory): 46 | """ 47 | Return ENV git configured for tests: 48 | 49 | 1. Both system and user configs are ignored 50 | 2. Custom git user 51 | 3. .gitmodules file is ignored by default 52 | """ 53 | e = { 54 | 'GIT_CONFIG_NOSYSTEM': 'true', 55 | 'HOME': tmpdir_factory.getbasetemp().strpath 56 | } 57 | 58 | with tmpdir_factory.getbasetemp().join('.gitconfig').open('wb+') as f: 59 | f.writelines([ 60 | b'[core]\n', 61 | 'attributesfile = {0}\n'.format(as_posix(tmpdir_factory.getbasetemp().join('.gitattributes').strpath)).encode(), 62 | b'[user]\n', 63 | b'name = git-archive-all\n', 64 | b'email = git-archive-all@example.com\n', 65 | ]) 66 | 67 | # .gitmodules's content is dynamic and is maintained by git. 68 | # It's therefore ignored solely to simplify tests. 69 | # 70 | # If test is run with the --no-exclude CLI option (or its exclude=False API equivalent) 71 | # then the file itself is included while its content is discarded for the same reason. 72 | with tmpdir_factory.getbasetemp().join('.gitattributes').open('wb+') as f: 73 | f.writelines([ 74 | b'.gitmodules export-ignore\n' 75 | ]) 76 | 77 | return e 78 | 79 | 80 | class Record: 81 | def __init__(self, kind, contents, excluded=False): 82 | self.kind = kind 83 | self.contents = contents 84 | self.excluded = excluded 85 | 86 | def __getitem__(self, item): 87 | return self.contents[item] 88 | 89 | def __setitem__(self, key, value): 90 | self.contents[key] = value 91 | 92 | 93 | FileRecord = partial(Record, 'file', excluded=False) 94 | DirRecord = partial(Record, 'dir', excluded=False) 95 | SubmoduleRecord = partial(Record, 'submodule', excluded=False) 96 | 97 | 98 | class Repo: 99 | def __init__(self, path): 100 | self.path = os.path.abspath(fspath(path)) 101 | 102 | def init(self): 103 | os.mkdir(self.path) 104 | check_call(['git', 'init'], cwd=self.path) 105 | 106 | def add(self, rel_path, record): 107 | if record.kind == 'file': 108 | return self.add_file(rel_path, record.contents) 109 | elif record.kind == 'dir': 110 | return self.add_dir(rel_path, record.contents) 111 | elif record.kind == 'submodule': 112 | return self.add_submodule(rel_path, record.contents) 113 | else: 114 | raise ValueError 115 | 116 | def add_file(self, rel_path, contents): 117 | file_path = os_path_join(self.path, rel_path) 118 | 119 | with open(file_path, 'wb') as f: 120 | f.write(contents) 121 | 122 | check_call(['git', 'add', as_posix(os.path.normpath(file_path))], cwd=self.path) 123 | return file_path 124 | 125 | def add_dir(self, rel_path, contents): 126 | dir_path = os_path_join(self.path, rel_path) 127 | makedirs(dir_path) 128 | 129 | for k, v in contents.items(): 130 | self.add(as_posix(os.path.normpath(os_path_join(dir_path, k))), v) 131 | 132 | check_call(['git', 'add', dir_path], cwd=self.path) 133 | return dir_path 134 | 135 | def add_submodule(self, rel_path, contents): 136 | submodule_path = os_path_join(self.path, rel_path) 137 | r = Repo(submodule_path) 138 | r.init() 139 | r.add_dir('.', contents) 140 | r.commit('init') 141 | check_call(['git', 'submodule', 'add', as_posix(os.path.normpath(submodule_path))], cwd=self.path) 142 | return submodule_path 143 | 144 | def commit(self, message): 145 | check_call(['git', 'commit', '-m', 'init'], cwd=self.path) 146 | 147 | def archive(self, path, exclude=True): 148 | a = GitArchiver(exclude=exclude, main_repo_abspath=self.path) 149 | a.create(path) 150 | 151 | 152 | def make_expected_tree(contents, exclude=True): 153 | e = {} 154 | 155 | for k, v in contents.items(): 156 | if v.kind == 'file' and not (exclude and v.excluded): 157 | e[k] = v.contents 158 | elif v.kind in ('dir', 'submodule') and not (exclude and v.excluded): 159 | # See the comment in git_env. 160 | if v.kind == 'submodule' and not exclude: 161 | e['.gitmodules'] = None 162 | 163 | for nested_k, nested_v in make_expected_tree(v.contents, exclude).items(): 164 | nested_k = as_posix(os_path_join(k, nested_k)) 165 | e[nested_k] = nested_v 166 | 167 | return e 168 | 169 | 170 | def make_actual_tree(tar_file): 171 | a = {} 172 | 173 | for m in tar_file.getmembers(): 174 | if m.isfile(): 175 | name = fspath(m.name) 176 | 177 | # See the comment in git_env. 178 | if not name.endswith(fspath('.gitmodules')): 179 | a[name] = tar_file.extractfile(m).read() 180 | else: 181 | a[name] = None 182 | else: 183 | raise NotImplementedError 184 | 185 | return a 186 | 187 | 188 | base = { 189 | 'app': DirRecord({ 190 | '__init__.py': FileRecord(b'#Beautiful is better than ugly.'), 191 | }), 192 | 'lib': SubmoduleRecord({ 193 | '__init__.py': FileRecord(b'#Explicit is better than implicit.'), 194 | 'extra': SubmoduleRecord({ 195 | '__init__.py': FileRecord(b'#Simple is better than complex.'), 196 | }) 197 | }) 198 | } 199 | 200 | base_quoted = deepcopy(base) 201 | base_quoted['data'] = DirRecord({ 202 | '\"hello world.dat\"': FileRecord(b'Special cases aren\'t special enough to break the rules.'), 203 | '\'hello world.dat\'': FileRecord(b'Although practicality beats purity.') 204 | }) 205 | 206 | ignore_in_root = deepcopy(base) 207 | ignore_in_root['.gitattributes'] = FileRecord(b'tests/__init__.py export-ignore') 208 | ignore_in_root['tests'] = DirRecord({ 209 | '__init__.py': FileRecord(b'#Complex is better than complicated.', excluded=True) 210 | }) 211 | 212 | ignore_in_submodule = deepcopy(base) 213 | ignore_in_submodule['lib']['.gitattributes'] = FileRecord(b'tests/__init__.py export-ignore') 214 | ignore_in_submodule['lib']['tests'] = DirRecord({ 215 | '__init__.py': FileRecord(b'#Complex is better than complicated.', excluded=True) 216 | }) 217 | 218 | ignore_in_nested_submodule = deepcopy(base) 219 | ignore_in_nested_submodule['lib']['extra']['.gitattributes'] = FileRecord(b'tests/__init__.py export-ignore') 220 | ignore_in_nested_submodule['lib']['extra']['tests'] = DirRecord({ 221 | '__init__.py': FileRecord(b'#Complex is better than complicated.', excluded=True) 222 | }) 223 | 224 | ignore_in_submodule_from_root = deepcopy(base) 225 | ignore_in_submodule_from_root['.gitattributes'] = FileRecord(b'lib/tests/__init__.py export-ignore') 226 | ignore_in_submodule_from_root['lib']['tests'] = DirRecord({ 227 | '__init__.py': FileRecord(b'#Complex is better than complicated.', excluded=True) 228 | }) 229 | 230 | ignore_in_nested_submodule_from_root = deepcopy(base) 231 | ignore_in_nested_submodule_from_root['.gitattributes'] = FileRecord(b'lib/extra/tests/__init__.py export-ignore') 232 | ignore_in_nested_submodule_from_root['lib']['extra']['tests'] = DirRecord({ 233 | '__init__.py': FileRecord(b'#Complex is better than complicated.', excluded=True) 234 | }) 235 | 236 | ignore_in_nested_submodule_from_submodule = deepcopy(base) 237 | ignore_in_nested_submodule_from_submodule['lib']['.gitattributes'] = FileRecord(b'extra/tests/__init__.py export-ignore') 238 | ignore_in_nested_submodule_from_submodule['lib']['extra']['tests'] = DirRecord({ 239 | '__init__.py': FileRecord(b'#Complex is better than complicated.', excluded=True) 240 | }) 241 | 242 | unset_export_ignore = deepcopy(base) 243 | unset_export_ignore['.gitattributes'] = FileRecord(b'.* export-ignore\n*.htaccess -export-ignore', excluded=True) 244 | unset_export_ignore['.a'] = FileRecord(b'Flat is better than nested.', excluded=True) 245 | unset_export_ignore['.b'] = FileRecord(b'Sparse is better than dense.', excluded=True) 246 | unset_export_ignore['.htaccess'] = FileRecord(b'Readability counts.') 247 | 248 | unicode_base = deepcopy(base) 249 | unicode_base['data'] = DirRecord({ 250 | 'مرحبا بالعالم.dat': FileRecord(b'Special cases aren\'t special enough to break the rules.') 251 | }) 252 | 253 | unicode_quoted = deepcopy(base) 254 | unicode_quoted['data'] = DirRecord({ 255 | '\"مرحبا بالعالم.dat\"': FileRecord(b'Special cases aren\'t special enough to break the rules.'), 256 | '\'привет мир.dat\'': FileRecord(b'Although practicality beats purity.') 257 | }) 258 | 259 | brackets_base = deepcopy(base) 260 | brackets_base['data'] = DirRecord({ 261 | '[.dat': FileRecord(b'Special cases aren\'t special enough to break the rules.'), 262 | '(.dat': FileRecord(b'Although practicality beats purity.'), 263 | '{.dat': FileRecord(b'Errors should never pass silently.'), 264 | '].dat': FileRecord(b'Unless explicitly silenced.'), 265 | ').dat': FileRecord(b'In the face of ambiguity, refuse the temptation to guess.'), 266 | '}.dat': FileRecord(b'There should be one-- and preferably only one --obvious way to do it.'), 267 | '[].dat': FileRecord(b'Although that way may not be obvious at first unless you\'re Dutch.'), 268 | '().dat': FileRecord(b'Now is better than never.'), 269 | '{}.dat': FileRecord(b'Although never is often better than *right* now.'), 270 | }) 271 | 272 | brackets_quoted = deepcopy(base) 273 | brackets_quoted['data'] = DirRecord({ 274 | '\"[.dat\"': FileRecord(b'Special cases aren\'t special enough to break the rules.'), 275 | '\'[.dat\'': FileRecord(b'Special cases aren\'t special enough to break the rules.'), 276 | '\"(.dat\"': FileRecord(b'Although practicality beats purity.'), 277 | '\'(.dat\'': FileRecord(b'Although practicality beats purity.'), 278 | '\"{.dat\"': FileRecord(b'Errors should never pass silently.'), 279 | '\'{.dat\'': FileRecord(b'Errors should never pass silently.'), 280 | '\"].dat\"': FileRecord(b'Unless explicitly silenced.'), 281 | '\'].dat\'': FileRecord(b'Unless explicitly silenced.'), 282 | '\").dat\"': FileRecord(b'In the face of ambiguity, refuse the temptation to guess.'), 283 | '\').dat\'': FileRecord(b'In the face of ambiguity, refuse the temptation to guess.'), 284 | '\"}.dat\"': FileRecord(b'There should be one-- and preferably only one --obvious way to do it.'), 285 | '\'}.dat\'': FileRecord(b'There should be one-- and preferably only one --obvious way to do it.'), 286 | '\"[].dat\"': FileRecord(b'Although that way may not be obvious at first unless you\'re Dutch.'), 287 | '\'[].dat\'': FileRecord(b'Although that way may not be obvious at first unless you\'re Dutch.'), 288 | '\"().dat\"': FileRecord(b'Now is better than never.'), 289 | '\'().dat\'': FileRecord(b'Now is better than never.'), 290 | '\"{}.dat\"': FileRecord(b'Although never is often better than *right* now.'), 291 | '\'{}.dat\'': FileRecord(b'Although never is often better than *right* now.'), 292 | }) 293 | 294 | quote_base = deepcopy(base) 295 | quote_base['data'] = DirRecord({ 296 | '\'.dat': FileRecord(b'Special cases aren\'t special enough to break the rules.'), 297 | '\".dat': FileRecord(b'Although practicality beats purity.'), 298 | }) 299 | 300 | quote_quoted = deepcopy(base) 301 | quote_quoted['data'] = DirRecord({ 302 | '\"\'.dat\"': FileRecord(b'Special cases aren\'t special enough to break the rules.'), 303 | '\'\'.dat\'': FileRecord(b'Special cases aren\'t special enough to break the rules.'), 304 | '\"\".dat\"': FileRecord(b'Although practicality beats purity.'), 305 | '\'\".dat\'': FileRecord(b'Although practicality beats purity.'), 306 | }) 307 | 308 | nonunicode_base = deepcopy(base) 309 | nonunicode_base['data'] = DirRecord({ 310 | b'test.\xc2': FileRecord(b'Special cases aren\'t special enough to break the rules.'), 311 | }) 312 | 313 | nonunicode_quoted = deepcopy(base) 314 | nonunicode_quoted['data'] = DirRecord({ 315 | b'\'test.\xc2\'': FileRecord(b'Special cases aren\'t special enough to break the rules.'), 316 | b'\"test.\xc2\"': FileRecord(b'Although practicality beats purity.'), 317 | }) 318 | 319 | backslash_base = deepcopy(base) 320 | backslash_base['data'] = DirRecord({ 321 | '\\.dat': FileRecord(b'Special cases aren\'t special enough to break the rules.'), 322 | }) 323 | 324 | backslash_quoted = deepcopy(base) 325 | backslash_quoted['data'] = DirRecord({ 326 | '\'\\.dat\'': FileRecord(b'Special cases aren\'t special enough to break the rules.'), 327 | '\"\\.dat\"': FileRecord(b'Although practicality beats purity.') 328 | }) 329 | 330 | non_unicode_backslash_base = deepcopy(base) 331 | non_unicode_backslash_base['data'] = DirRecord({ 332 | b'\\\xc2.dat': FileRecord(b'Special cases aren\'t special enough to break the rules.'), 333 | }) 334 | 335 | non_unicode_backslash_quoted = deepcopy(base) 336 | non_unicode_backslash_quoted['data'] = DirRecord({ 337 | b'\'\\\xc2.dat\'': FileRecord(b'Special cases aren\'t special enough to break the rules.'), 338 | b'\"\\\xc2.dat\"': FileRecord(b'Although practicality beats purity.') 339 | }) 340 | 341 | ignore_dir = { 342 | '.gitattributes': FileRecord(b'.gitattributes export-ignore\n**/src export-ignore\ndata/src/__main__.py -export-ignore', excluded=True), 343 | '__init__.py': FileRecord(b'#Beautiful is better than ugly.'), 344 | 'data': DirRecord({ 345 | 'src': DirRecord({ 346 | '__init__.py': FileRecord(b'#Explicit is better than implicit.', excluded=True), 347 | '__main__.py': FileRecord(b'#Simple is better than complex.') 348 | }) 349 | }) 350 | } 351 | 352 | skipif_file_darwin = pytest.mark.skipif(sys.platform.startswith('darwin'), reason='Invalid macOS filename.') 353 | skipif_file_win32 = pytest.mark.skipif(sys.platform.startswith('win32'), reason="Invalid Windows filename.") 354 | 355 | 356 | @pytest.mark.parametrize('contents', [ 357 | pytest.param(base, id='No Ignore'), 358 | pytest.param(base_quoted, id='No Ignore (Quoted)', marks=skipif_file_win32), 359 | pytest.param(ignore_in_root, id='Ignore in Root'), 360 | pytest.param(ignore_in_submodule, id='Ignore in Submodule'), 361 | pytest.param(ignore_in_nested_submodule, id='Ignore in Nested Submodule'), 362 | pytest.param(ignore_in_submodule_from_root, id='Ignore in Submodule from Root'), 363 | pytest.param(ignore_in_nested_submodule_from_root, id='Ignore in Nested Submodule from Root'), 364 | pytest.param(ignore_in_nested_submodule_from_submodule, id='Ignore in Nested Submodule from Submodule'), 365 | pytest.param(unset_export_ignore, id='-export-ignore'), 366 | pytest.param(unicode_base, id='Unicode'), 367 | pytest.param(unicode_quoted, id='Unicode (Quoted)', marks=skipif_file_win32), 368 | pytest.param(brackets_base, id='Brackets'), 369 | pytest.param(brackets_quoted, id="Brackets (Quoted)", marks=skipif_file_win32), 370 | pytest.param(quote_base, id="Quote", marks=skipif_file_win32), 371 | pytest.param(quote_quoted, id="Quote (Quoted)", marks=skipif_file_win32), 372 | pytest.param(nonunicode_base, id="Non-Unicode", marks=[skipif_file_win32, skipif_file_darwin]), 373 | pytest.param(nonunicode_quoted, id="Non-Unicode (Quoted)", marks=[skipif_file_win32, skipif_file_darwin]), 374 | pytest.param(backslash_base, id='Backslash', marks=skipif_file_win32), 375 | pytest.param(backslash_quoted, id='Backslash (Quoted)', marks=skipif_file_win32), 376 | pytest.param(non_unicode_backslash_base, id='Non-Unicode Backslash', marks=[skipif_file_win32, skipif_file_darwin]), 377 | pytest.param(non_unicode_backslash_quoted, id='Non-Unicode Backslash (Quoted)', marks=[skipif_file_win32, skipif_file_darwin]), 378 | pytest.param(ignore_dir, id='Ignore Directory') 379 | ]) 380 | @pytest.mark.parametrize('exclude', [ 381 | pytest.param(True, id='With export-ignore'), 382 | pytest.param(False, id='Without export-ignore'), 383 | ]) 384 | def test_ignore(contents, exclude, tmpdir, git_env, monkeypatch): 385 | """ 386 | Ensure that GitArchiver respects export-ignore. 387 | """ 388 | # On Python 2.7 contained code raises pytest.PytestWarning warning for no good reason. 389 | with warnings.catch_warnings(): 390 | warnings.simplefilter("ignore") 391 | 392 | for name, value in git_env.items(): 393 | monkeypatch.setenv(name, value) 394 | 395 | repo_path = os_path_join(tmpdir.strpath, 'repo') 396 | repo = Repo(repo_path) 397 | repo.init() 398 | repo.add_dir('.', contents) 399 | repo.commit('init') 400 | 401 | repo_tar_path = os_path_join(tmpdir.strpath, 'repo.tar') 402 | repo.archive(repo_tar_path, exclude=exclude) 403 | 404 | with TarFile(repo_tar_path, format=PAX_FORMAT, encoding='utf-8') as repo_tar: 405 | expected = make_expected_tree(contents, exclude) 406 | actual = make_actual_tree(repo_tar) 407 | 408 | assert actual == expected 409 | 410 | 411 | def test_cli(tmpdir, git_env, monkeypatch): 412 | contents = base 413 | 414 | # On Python 2.7 contained code raises pytest.PytestWarning warning for no good reason. 415 | with warnings.catch_warnings(): 416 | warnings.simplefilter("ignore") 417 | 418 | for name, value in git_env.items(): 419 | monkeypatch.setenv(name, value) 420 | 421 | repo_path = os_path_join(tmpdir.strpath, 'repo') 422 | repo = Repo(repo_path) 423 | repo.init() 424 | repo.add_dir('.', contents) 425 | repo.commit('init') 426 | 427 | repo_tar_path = os_path_join(tmpdir.strpath, 'repo.tar') 428 | git_archive_all.main(['git_archive_all.py', '--prefix', '', '-C', repo_path, repo_tar_path]) 429 | 430 | with TarFile(repo_tar_path, format=PAX_FORMAT, encoding='utf-8') as repo_tar: 431 | expected = make_expected_tree(contents) 432 | actual = make_actual_tree(repo_tar) 433 | 434 | assert actual == expected 435 | 436 | 437 | @pytest.mark.parametrize('version', [ 438 | b'git version 2.21.0.0.1', 439 | b'git version 2.21.0.windows.1' 440 | ]) 441 | def test_git_version_parse(version, mocker): 442 | mocker.patch.object(GitArchiver, 'run_git_shell', return_value=version) 443 | assert GitArchiver.get_git_version() == (2, 21, 0, 0, 1) 444 | 445 | 446 | def test_pycodestyle(): 447 | style = pycodestyle.StyleGuide(repeat=True, max_line_length=240) 448 | report = style.check_files(['git_archive_all.py']) 449 | assert report.total_errors == 0, "Found code style errors (and warnings)." 450 | -------------------------------------------------------------------------------- /travis-requirements.txt: -------------------------------------------------------------------------------- 1 | tox==3.20.1 2 | codecov==2.1.10 3 | tox-pyenv==1.1.0 4 | virtualenv==15.2.0 5 | --------------------------------------------------------------------------------