├── .flake8 ├── .github └── workflows │ └── main.yml ├── .gitignore ├── .hound.yml ├── .pylintrc ├── .pylinttestsrc ├── CONTRIBUTING.md ├── COPYING ├── GNUmakefile ├── KankuFile ├── README.md ├── TESTING.md ├── TarSCM ├── __init__.py ├── archive.py ├── changes.py ├── cli.py ├── config.py ├── exceptions.py ├── helpers.py ├── scm │ ├── __init__.py │ ├── base.py │ ├── bzr.py │ ├── git.py │ ├── hg.py │ ├── svn.py │ └── tar.py └── tasks.py ├── appimage ├── appimage.service ├── debian ├── changelog ├── compat ├── control ├── copyright ├── rules └── source │ └── format ├── dist ├── debian.dsc └── obs-service-tar_scm.spec ├── obs_gbp ├── obs_scm ├── requirements.txt ├── snapcraft ├── snapcraft.service ├── tar ├── tar.service ├── tar_scm.py ├── tar_scm.rc ├── tar_scm.service.in └── tests ├── __init__.py ├── archiveobscpiotestcases.py ├── bzrfixtures.py ├── bzrtests.py ├── commontests.py ├── fake_classes.py ├── fixtures.py ├── fixtures ├── ArchiveOBSCpioTestCases │ ├── test_obscpio_broken_link │ │ └── repo │ │ │ └── broken_link │ ├── test_obscpio_create_archive │ │ └── repo │ │ │ ├── Readme.md │ │ │ ├── a │ │ │ ├── dir1 │ │ │ └── .keep │ │ │ └── test.spec │ ├── test_obscpio_extract_d │ │ └── repo │ │ │ ├── Readme.md │ │ │ ├── a │ │ │ ├── dir1 │ │ │ └── .keep │ │ │ └── test.spec │ ├── test_obscpio_extract_glob │ │ └── repo │ │ │ ├── Readme.md │ │ │ ├── a │ │ │ ├── dir1 │ │ │ └── .keep │ │ │ ├── test.rpmlintrc │ │ │ └── test.spec │ ├── test_obscpio_extract_mf │ │ └── repo │ │ │ ├── Readme.md │ │ │ ├── a │ │ │ ├── dir1 │ │ │ └── .keep │ │ │ └── test.spec │ └── test_obscpio_extract_of │ │ └── repo │ │ ├── Readme.md │ │ ├── a │ │ ├── dir1 │ │ └── .keep │ │ └── test.spec ├── GitTests │ └── test_find_valid_commit │ │ └── fixtures.tar ├── TasksTestCases │ ├── test_appimage_empty_build │ │ └── appimage.yml │ ├── test_appimage_empty_build_git │ │ └── appimage.yml │ ├── test_generate_tl_multi_tasks │ │ └── snapcraft.yaml │ ├── test_generate_tl_single_task │ │ └── snapcraft.yaml │ ├── test_generate_tl_st_appimage │ │ └── appimage.yml │ └── test_tasks_finalize └── UnitTestCases │ ├── test_config_debug_tar_scm │ └── test.rc │ ├── test_config_files_ordering │ ├── a.cfg │ └── b.cfg │ ├── test_config_no_faked_header │ └── test.ini │ └── test_unicode_in_filename │ └── test │ └── äöü.txt ├── gitfixtures.py ├── githgtests.py ├── gitsvntests.py ├── gittests.py ├── hgfixtures.py ├── hgtests.py ├── scm-wrapper ├── scm.py ├── scmlogs.py ├── svnfixtures.py ├── svntests.py ├── tarfixtures.py ├── tartests.py ├── tasks.py ├── test.py ├── testassertions.py ├── testenv.py ├── unittestcases.py └── utils.py /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | exclude = tests 3 | ignore=E221,E251,E272,E241,E731,F401,E722 4 | -------------------------------------------------------------------------------- /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | # This is a basic workflow to help you get started with Actions 2 | 3 | name: Testing 4 | # Controls when the workflow will run 5 | on: 6 | # Triggers the workflow on push or pull request events but only for the master branch 7 | push: 8 | branches: [ master ] 9 | pull_request: 10 | branches: [ master ] 11 | 12 | # Allows you to run this workflow manually from the Actions tab 13 | workflow_dispatch: 14 | 15 | # A workflow run is made up of one or more jobs that can run sequentially or in parallel 16 | jobs: 17 | # This workflow contains a single job called "build" 18 | build: 19 | # The type of runner that the job will run on 20 | runs-on: ubuntu-latest 21 | strategy: 22 | matrix: 23 | python-version: [ '3.8', '3.9', '3.10', '3.11', '3.12', '3.13'] 24 | 25 | # Steps represent a sequence of tasks that will be executed as part of the job 26 | steps: 27 | # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it 28 | - uses: actions/checkout@v3 29 | 30 | - name: Install required dpkg packages 31 | run: sudo apt-get install libxslt1-dev bzr subversion mercurial 32 | 33 | - name: Generate default locales 34 | run: | 35 | sudo apt-get update && sudo apt-get install tzdata locales -y && sudo locale-gen "en_US.UTF-8" 36 | sudo dpkg-reconfigure locales 37 | sudo update-locale "LANG=en_US.UTF-8" 38 | sudo update-locale "LC_ALL=en_US.UTF-8" 39 | locale 40 | 41 | - name: Setup python 42 | uses: actions/setup-python@v4 43 | with: 44 | python-version: ${{ matrix.python-version }} 45 | allow-prereleases: true 46 | 47 | - name: Install python dependencies 48 | run: pip install -r requirements.txt 49 | 50 | - name: Lint with flake8 51 | run: make flake8 52 | 53 | - name: Lint code with pylint 54 | run: make pylint 55 | 56 | - name: Lint test with pylint 57 | run: make pylinttest 58 | 59 | - name: Run tests 60 | run: | 61 | locale 62 | make test 63 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | tar_scm 2 | *.pyc 3 | .*.sw? 4 | tmp/ 5 | .coverage 6 | htmlcov/ 7 | test*.log 8 | cover*.log 9 | -------------------------------------------------------------------------------- /.hound.yml: -------------------------------------------------------------------------------- 1 | flake8: 2 | enabled: true 3 | config_file: .flake8 4 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to `tar_scm` 2 | 3 | If you notice an issue, please first check [the list of known issues](https://github.com/openSUSE/obs-service-tar_scm/issues?state=open). 4 | 5 | It is a known issue that currently [we need better developer 6 | documentation](https://github.com/openSUSE/obs-service-tar_scm/issues/195), 7 | and help fixing this is extremely welcome. 8 | 9 | ## Pull requests 10 | 11 | [Pull requests](https://help.github.com/articles/using-pull-requests) 12 | are extremely welcome, but will not be accepted unless they contain 13 | corresponding additions/modifications to the test suite - test suite 14 | bit-rot is the path to gloom and despair :-) [`TESTING.md`](TESTING.md) 15 | explains how the test suite works. 16 | 17 | Before starting work on a pull request, please read this article 18 | describing [7 principles for contributing patches to software projects](http://blog.adamspiers.org/2012/11/10/7-principles-for-contributing-patches-to-software-projects/). 19 | 20 | ## Bug reports 21 | 22 | If you are unable to provide a fix via a pull request, please 23 | [submit an issue](https://github.com/openSUSE/obs-service-tar_scm/issues/new). 24 | 25 | Many thanks in advance! 26 | -------------------------------------------------------------------------------- /GNUmakefile: -------------------------------------------------------------------------------- 1 | # Ensure that we don't accidentally ignore failing commands just 2 | # because they're in a pipeline. This applies in particular to piping 3 | # the test runs through tee(1). 4 | # http://stackoverflow.com/a/31605520/179332 5 | SHELL = /bin/bash -o pipefail 6 | 7 | DESTDIR ?= 8 | PREFIX = /usr 9 | SYSCFG = /etc 10 | 11 | CLEAN_PYFILES = \ 12 | ./tar_scm.py \ 13 | ./TarSCM/scm/bzr.py \ 14 | ./TarSCM/scm/svn.py \ 15 | ./TarSCM/exceptions.py \ 16 | 17 | CLEAN_TEST_PYFILES = \ 18 | ./tests/__init__.py \ 19 | ./tests/utils.py \ 20 | ./tests/tasks.py \ 21 | ./tests/fake_classes.py \ 22 | ./tests/unittestcases.py \ 23 | ./tests/archiveobscpiotestcases.py \ 24 | ./tests/gittests.py \ 25 | ./tests/fixtures.py \ 26 | ./tests/bzrfixtures.py \ 27 | ./tests/gitfixtures.py \ 28 | ./tests/hgfixtures.py \ 29 | ./tests/svnfixtures.py \ 30 | ./tests/tarfixtures.py \ 31 | ./tests/commontests.py \ 32 | ./tests/bzrtests.py \ 33 | ./tests/svntests.py \ 34 | 35 | PYLINT_READY_TEST_MODULES = \ 36 | $(CLEAN_TEST_PYFILES) \ 37 | ./tests/test.py \ 38 | ./tests/scmlogs.py \ 39 | ./tests/tartests.py \ 40 | 41 | PYLINT_READY_MODULES = \ 42 | $(CLEAN_PYFILES) \ 43 | ./TarSCM/__init__.py \ 44 | ./TarSCM/scm/git.py \ 45 | ./TarSCM/scm/hg.py \ 46 | ./TarSCM/scm/__init__.py \ 47 | ./TarSCM/cli.py \ 48 | ./TarSCM/tasks.py \ 49 | 50 | define first_in_path 51 | $(or \ 52 | $(firstword $(wildcard \ 53 | $(foreach p,$(1),$(addsuffix /$(p),$(subst :, ,$(PATH)))) \ 54 | )), \ 55 | $(error Need one of: $(1)) \ 56 | ) 57 | endef 58 | 59 | define first_in_path_opt 60 | $(or \ 61 | $(firstword $(wildcard \ 62 | $(foreach p,$(1),$(addsuffix /$(p),$(subst :, ,$(PATH)))) \ 63 | )), \ 64 | ) 65 | endef 66 | 67 | ALL_PYTHON3 = python3.9 python3.8 python3.7 python-3.7 python3.6 python-3.6 python3.5 python-3.5 python3.4 python-3.4 python3.3 python-3.3 python3.2 python-3.2 python3 68 | ALL_PYTHON2 = python2.7 python-2.7 python2.6 python-2.6 python2 69 | 70 | # Ensure that correct python version is used in travis 71 | y = $(subst ., ,$(TRAVIS_PYTHON_VERSION)) 72 | PYTHON_MAJOR := $(word 1, $(y)) 73 | ifeq ($(PYTHON_MAJOR), 2) 74 | ALL_PYTHONS = $(ALL_PYTHON2) python 75 | else 76 | ALL_PYTHONS = $(ALL_PYTHON3) $(ALL_PYTHON2) python 77 | endif 78 | 79 | PYTHON = $(call first_in_path,$(ALL_PYTHONS)) 80 | PYTHON2 = $(call first_in_path,$(ALL_PYTHON2)) 81 | 82 | mylibdir = $(PREFIX)/lib/obs/service 83 | mycfgdir = $(SYSCFG)/obs/services 84 | 85 | LIST_PY_FILES=git ls-tree --name-only -r HEAD | grep '\.py$$' 86 | PY_FILES=$(shell $(LIST_PY_FILES)) 87 | 88 | ALL_PYLINT3 = pylint-3.4 pylint3.4 pylint-3.5 pylint3.5 pylint-3.6 pylint3.6 pylint-3.7 pylint3.7 pylint-3.8 pylint 89 | ALL_FLAKE83 = flake8-3.6 flake8-36 flake8-37 flake8-3.7 flake8 90 | 91 | PYLINT3 = $(call first_in_path_opt,$(ALL_PYLINT3)) 92 | 93 | FLAKE83 = $(call first_in_path_opt,$(ALL_FLAKE83)) 94 | 95 | python_version_full := $(wordlist 2,4,$(subst ., ,$(shell python --version 2>&1))) 96 | python_version_major := $(word 1,${python_version_full}) 97 | 98 | all: check 99 | 100 | .PHONY: check 101 | check: check3 test2 102 | 103 | .PHONY: check3 104 | check3: flake8 pylint pylinttest test3 105 | 106 | .PHONY: list-py-files 107 | list-py-files: 108 | @$(LIST_PY_FILES) 109 | 110 | .PHONY: flake8 111 | flake8: 112 | @if [ "x$(python_version_major)" == "x2" -a -n "$(CI)" ]; then \ 113 | echo "Skipping flake8 - python2 in CI" \ 114 | ;else \ 115 | if [ "x$(FLAKE83)" != "x" ]; then \ 116 | echo "Running flake83"\ 117 | $(FLAKE83)\ 118 | echo "Finished flake83"\ 119 | else \ 120 | echo "flake8 for python3 not found or python major version == 2 ($(python_version_major))";\ 121 | fi \ 122 | fi 123 | 124 | 125 | .PHONY: test2 126 | test2: 127 | : Running the test suite. Please be patient - this takes a few minutes ... 128 | TAR_SCM_TESTMODE=1 PYTHONPATH=. $(PYTHON2) tests/test.py 2>&1 | tee ./test.log 129 | 130 | .PHONY: test3 131 | test3: 132 | : Running the test suite. Please be patient - this takes a few minutes ... 133 | TAR_SCM_TESTMODE=1 PYTHONPATH=. python3 tests/test.py 2>&1 | tee ./test3.log 134 | 135 | .PHONY: test 136 | test: 137 | : Running the test suite. Please be patient - this takes a few minutes ... 138 | TAR_SCM_TESTMODE=1 PYTHONPATH=. python tests/test.py 2>&1 | tee ./test3.log 139 | 140 | .PHONY: pylint 141 | pylint: 142 | @if [ "x$(python_version_major)" == "x2" -a -n "$(CI)" ]; then \ 143 | echo "Skipping pylint - python2 in CI" \ 144 | ;else \ 145 | if [ "x$(PYLINT3)" != "x" ]; then \ 146 | $(PYLINT3) --rcfile=./.pylintrc $(PYLINT_READY_MODULES); \ 147 | PYTHONPATH=tests $(PYLINT3) --rcfile=./.pylintrc $(PYLINT_READY_MODULES); \ 148 | else \ 149 | echo "PYLINT3 not set or python major version == 2 ($(python_version_major)) - Skipping tests!"; \ 150 | fi \ 151 | fi 152 | 153 | .PHONY: pylinttest 154 | pylinttest: 155 | @if [ "x$(python_version_major)" == "x2" -a -n "$(CI)" ]; then \ 156 | echo "Skipping pylinttest - python2 in CI" \ 157 | ;else \ 158 | if [ "x$(PYLINT3)" != "x" ]; then \ 159 | PYTHONPATH=tests $(PYLINT3) --rcfile=./.pylinttestsrc $(PYLINT_READY_TEST_MODULES); \ 160 | else \ 161 | echo "PYLINT3 not set or python major version == 2 ($(python_version_major)) - Skip linting tests!"; \ 162 | fi \ 163 | fi 164 | 165 | 166 | cover: 167 | PYTHONPATH=. coverage2 run tests/test.py 2>&1 | tee ./cover.log 168 | coverage2 html --include=./TarSCM/* 169 | 170 | .PHONY: tar_scm 171 | tar_scm: tar_scm.py 172 | @echo "Creating $@ which uses $(PYTHON) ..." 173 | sed 's,^\#!/usr/bin/.*,#!$(PYTHON),' $< > $@ 174 | 175 | COMPILED_PYTHON = true 176 | 177 | .PHONY: install 178 | 179 | install: dirs tar_scm service $(if $(findstring $(COMPILED_PYTHON),true),compile) 180 | install -m 0755 tar_scm $(DESTDIR)$(mylibdir)/tar_scm 181 | install -m 0644 tar_scm.rc $(DESTDIR)$(mycfgdir)/tar_scm 182 | # Recreate links, otherwise reinstalling would fail 183 | [ ! -L $(DESTDIR)$(mylibdir)/obs_scm ] || rm $(DESTDIR)$(mylibdir)/obs_scm 184 | ln -s tar_scm $(DESTDIR)$(mylibdir)/obs_scm 185 | [ ! -L $(DESTDIR)$(mylibdir)/tar ] || rm $(DESTDIR)$(mylibdir)/tar 186 | ln -s tar_scm $(DESTDIR)$(mylibdir)/tar 187 | [ ! -L $(DESTDIR)$(mylibdir)/appimage ] || rm $(DESTDIR)$(mylibdir)/appimage 188 | ln -s tar_scm $(DESTDIR)$(mylibdir)/appimage 189 | [ ! -L $(DESTDIR)$(mylibdir)/snapcraft ] || rm $(DESTDIR)$(mylibdir)/snapcraft 190 | ln -s tar_scm $(DESTDIR)$(mylibdir)/snapcraft 191 | ifeq (1, ${WITH_GBP}) 192 | [ ! -L $(DESTDIR)$(mylibdir)/obs_gbp ] || rm $(DESTDIR)$(mylibdir)/obs_gbp 193 | ln -s tar_scm $(DESTDIR)$(mylibdir)/obs_gbp 194 | endif 195 | find ./TarSCM/ -name '*.py*' -exec install -D -m 644 {} $(DESTDIR)$(mylibdir)/{} \; 196 | 197 | .PHONY: dirs 198 | dirs: 199 | mkdir -p $(DESTDIR)$(mylibdir) 200 | mkdir -p $(DESTDIR)$(mylibdir)/TarSCM 201 | mkdir -p $(DESTDIR)$(mylibdir)/TarSCM/scm 202 | mkdir -p $(DESTDIR)$(mycfgdir) 203 | mkdir -p $(DESTDIR)$(mycfgdir)/tar_scm.d 204 | 205 | .PHONY: service 206 | service: dirs 207 | install -m 0644 tar.service $(DESTDIR)$(mylibdir)/ 208 | install -m 0644 snapcraft.service $(DESTDIR)$(mylibdir)/ 209 | install -m 0644 appimage.service $(DESTDIR)$(mylibdir)/ 210 | sed -e '/^===OBS_ONLY/,/^===/d' -e '/^===GBP_ONLY/,/^===/d' -e '/^===/d' tar_scm.service.in > $(DESTDIR)$(mylibdir)/tar_scm.service 211 | sed -e '/^===TAR_ONLY/,/^===/d' -e '/^===GBP_ONLY/,/^===/d' -e '/^===/d' tar_scm.service.in > $(DESTDIR)$(mylibdir)/obs_scm.service 212 | ifeq (1, ${WITH_GBP}) 213 | sed -e '/^===OBS_ONLY/,/^===/d' -e '/^===TAR_ONLY/,/^===/d' -e '/^===/d' tar_scm.service.in > $(DESTDIR)$(mylibdir)/obs_gbp.service 214 | endif 215 | 216 | show-python: 217 | @echo "$(PYTHON)" 218 | 219 | .PHONY: clean 220 | clean: 221 | find -name '*.pyc' -exec rm -f {} \; 222 | rm -rf ./tests/tmp/ 223 | rm -f ./test.log 224 | rm -f ./test3.log 225 | rm -f ./cover.log 226 | 227 | compile: 228 | find -name '*.py' -exec $(PYTHON) -m py_compile {} \; 229 | -------------------------------------------------------------------------------- /KankuFile: -------------------------------------------------------------------------------- 1 | # 2 | Kanku::Util::IPTables: 3 | start_port: 49001 4 | 5 | 6 | domain_name: obs-service-tar_scm 7 | default_job: tw 8 | login_user: root 9 | login_pass: kankudai 10 | 11 | qemu: 12 | user: 13 | 14 | jobs: 15 | tw: 16 | - 17 | use_module: Kanku::Handler::SetJobContext 18 | options: 19 | host_interface: eth0 20 | - 21 | use_module: Kanku::Handler::OBSCheck 22 | options: 23 | api_url: https://api.opensuse.org/public 24 | # Please have a look at 25 | # https://build.opensuse.org/project/show/devel:kanku:immages 26 | # to find more official Images 27 | project: devel:kanku:images 28 | package: openSUSE-Tumbleweed-JeOS:ext4 29 | repository: images_tumbleweed 30 | arch: x86_64 31 | - 32 | use_module: Kanku::Handler::ImageDownload 33 | options: 34 | use_cache: 1 35 | 36 | - 37 | use_module: Kanku::Handler::CreateDomain 38 | options: 39 | memory: 2G 40 | vcpu: 4 41 | use_9p: 1 42 | #forward_port_list: tcp:22,tcp:443 43 | - 44 | use_module: Kanku::Handler::PrepareSSH 45 | - 46 | use_module: Kanku::Handler::ExecuteCommandViaSSH 47 | options: 48 | commands: 49 | - zypper -n in git mercurial subversion make tar 50 | - zypper -n in python3-PyYAML python3-python-dateutil python3-pylint python3-flake8 python3-six 51 | - 52 | use_module: Kanku::Handler::ExecuteCommandViaSSH 53 | options: 54 | username: kanku 55 | commands: 56 | # test python3 57 | - TAR_SCM_TC=UnitTestCases,TasksTestCases,SCMBaseTestCases,GitTests,SvnTests,HgTests,TarTestCases make -C /tmp/kanku clean check3 58 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # tar_scm (OBS source service) [![Build Status](https://travis-ci.org/openSUSE/obs-service-tar_scm.png?branch=master)](https://travis-ci.org/openSUSE/obs-service-tar_scm) 2 | 3 | This is the git repository for 4 | [openSUSE:Tools/obs-service-tar_scm](https://build.opensuse.org/package/show/openSUSE:Tools/obs-service-tar_scm), 5 | which provides several [source 6 | services](http://openbuildservice.org/help/manuals/obs-user-guide/cha.obs.source_service.html) 7 | for the [Open Build Service](http://openbuildservice.org/) which all 8 | assist with packaging source code from SCM (source code management) 9 | repositories into tarballs. The authoritative source is 10 | https://github.com/openSUSE/obs-service-tar_scm. 11 | 12 | ## Services 13 | 14 | ### tar_scm *(deprecated)* 15 | 16 | `tar_scm` is the legacy source service used to create a source tarball 17 | from one of the supported SCM (source code management) tools: `git`, 18 | `hg`, `svn`, and `bzr`. 19 | 20 | `tar_scm` supports many options, e.g. it can adjust resulting tarball 21 | parameters, include or exclude particular files when creating the 22 | tarball, or generate an `rpm` changelog from the SCM commit log. For the 23 | full list of options please see `tar_scm.service.in`. 24 | 25 | Apart from various SCM like git, hg, bzr or svn, it additionally 26 | supports `--url` option that allows you to specify URL of the upstream 27 | tarball to be downloaded. 28 | 29 | `tar_scm` can be used in combination with other services like 30 | [download_files](https://github.com/openSUSE/obs-service-download_files), 31 | [recompress](https://github.com/openSUSE/obs-service-recompress) or 32 | [set_version](https://github.com/openSUSE/obs-service-set_version) 33 | e.g. within the [GIT integration](https://en.opensuse.org/openSUSE:Build_Service_Concept_SourceService#Example_2:_GIT_integration) 34 | workflow. 35 | 36 | **`tar_scm` is deprecated in favour of `obs_scm`.** 37 | 38 | ### obs_scm 39 | 40 | `obs_scm` is similar in concept to `tar_scm`, but instead of directly 41 | generating tarballs, it instead uses the new `obscpio` archive format 42 | (see below) as an intermediate space-efficient format in which to 43 | store the sources. 44 | 45 | **It is recommended to use `obs_scm` in favour to `tar_scm`**, because 46 | it provides the following advantages: 47 | 48 | 1. When you `osc checkout`, you'll also get a local checkout directory 49 | within the project directory, inside which you can develop as usual 50 | and test your changes with local builds, even without having to 51 | commit or push your changes anywhere. 52 | 53 | 2. It helps to save a *lot* of disk space on the server side, 54 | especially when used in continuous integration (e.g. nightly builds 55 | and builds of pull requests). 56 | 57 | The usual source tarballs can be regenerated from this at build-time 58 | using the `tar` and `recompress` source services, so no changes to 59 | `.spec` files are required when switching to `obs_scm` and `obscpio`. 60 | 61 | Having said that, it may be more efficient to drop the build-time 62 | usage of `recompress`, since at build-time `rpmbuild` would decompress 63 | the same file soon after compressing it. In this case, only the `tar` 64 | source service would be used to reconstruct an uncompressed tarball to 65 | be consumed by the `.spec` file. However this has the side-effect 66 | that the resulting `.src.rpm` will contain an uncompressed tarball 67 | too. This is not necessarily a problem because `.src.rpm` files are 68 | compressed anyway, and in fact it may even be *more* efficient to 69 | avoid double compression (i.e. the sources within the `.src.rpm`, and 70 | the `.src.rpm` itself both being compressed). But this depends very 71 | much on the combination of compression formats used for compression of 72 | the sources, and for compression of the `.src.rpm`. Therefore the 73 | decision whether to use `recompress` will depend on what format is 74 | desired within the resulting `.src.rpm`, and on the types of 75 | compression being used for both the tarball and by `rpmbuild` for 76 | constructing the source rpms. 77 | 78 | `obs_scm` additionally generates a file named `.obsinfo` 79 | which includes useful information from your SCM system, such as the 80 | name, version number, mtime, and commit SHA1. This data is then used 81 | by the `tar` service (see below) to reconstruct a tarball for use by 82 | `rpmbuild` at build-time, and also by the 83 | [`set_version`](https://github.com/openSUSE/obs-service-set_version) 84 | source service in order to set the version in build description files 85 | such as `.spec` or `.dsc` files. 86 | 87 | ### tar 88 | 89 | The `tar` source service creates a tarball out of a `.obscpio` archive 90 | and a corresponding `.obsinfo` file which contains metadata about it. 91 | Typically this service is run at build-time, e.g. 92 | 93 | 94 | 95 | since storing the `.tar` file in OBS would duplicate the source data 96 | in the `.obscpio` and defeat the point of using `.obscpio` in the 97 | first place, which is to save space on the OBS server. 98 | 99 | See http://openbuildservice.org/2016/04/08/new_git_in_27/ for an example 100 | combining usage of the `obs_scm` and `tar` source services. 101 | 102 | ### snapcraft 103 | 104 | The `snapcraft` source service can be used to fetch sources before 105 | building a [`snappy` app (a.k.a. *snap*)](https://snapcraft.io/). 106 | 107 | It parses [a `snapcraft.yaml` 108 | file](https://docs.snapcraft.io/build-snaps/syntax), looking for any 109 | [parts in the `parts` 110 | section](https://docs.snapcraft.io/build-snaps/parts) which have 111 | [`source-type`](https://docs.snapcraft.io/reference/plugins/source) 112 | set to one of the supported SCMs. For each one it will fetch the 113 | sources via the SCM from the upstream repository, and build a tarball 114 | from it. 115 | 116 | Finally it will write a new version of `snapcraft.yaml` which has the 117 | `source` value rewritten from the original URL, to the name of the 118 | part, which is also the name of the newly created local file. This 119 | allows the snap to be built purely from local files. 120 | 121 | ### appimage 122 | 123 | The `appimage` source service can be used to fetch sources before 124 | building an [AppImage](http://appimage.org/). It parses [an 125 | `appimage.yml` 126 | file](https://github.com/AppImage/AppImages/blob/master/YML.md), looks 127 | for an optional `build` section at the top-level, and for any sub-key 128 | named after a supported SCM, it will treat the corresponding value as 129 | a URL, fetch the sources via the SCM from the upstream repository, and 130 | build a tarball from it. You can find example `appimage.yml` files 131 | under the `tests/fixtures/` subdirectory. 132 | 133 | ### gbp 134 | The `obs_gbp` service can be used to create Debian source artefacts 135 | (.dsc, .orig.tar.gz and if non-native .debian.tar.gz or .diff.gz) from 136 | Git repositories, following the very popular [git-buildpackage workflow](https://honk.sigxcpu.org/piki/projects/git-buildpackage/). 137 | Requires git-buildpackage to be installed. 138 | 139 | ## Archive Formats 140 | 141 | ### tar 142 | 143 | The standard `tar` archive format is used as output format by the 144 | `tar` and `tar_scm` source services. 145 | 146 | ### obscpio 147 | 148 | `obscpio` archives are 149 | [`cpio`](https://www.gnu.org/software/cpio/manual/cpio.html) archives 150 | in `newc` format. Using these allows the [OBS Delta 151 | Store](http://openbuildservice.org/help/manuals/obs-reference-guide/cha.obs.architecture.html#delta_store) 152 | to store changes server-side in a space-efficient incremental way, 153 | independently of your chosen SCM. Then at build-time, the `tar` 154 | source service converts a file from this format into a regular `.tar` 155 | for use by `rpmbuild`. This is described in more detail in this blog 156 | post: 157 | 158 | - http://openbuildservice.org/2016/04/08/new_git_in_27/ 159 | 160 | ## Installation 161 | 162 | The files in this top-level directory need to be installed using the 163 | following: 164 | 165 | make install 166 | 167 | ## User documentation 168 | 169 | There isn't yet any comprehensive user documentation (see [issue 170 | #238](https://github.com/openSUSE/obs-service-tar_scm/issues/238)), 171 | but in the meantime, in addition to the information in this README, 172 | the following resources may be helpful: 173 | 174 | - The XML `.service` files which document the parameters for 175 | each source service: 176 | - the [`tar_scm.service.in` template](https://github.com/openSUSE/obs-service-tar_scm/blob/master/tar_scm.service.in) 177 | which is used to generate `tar_scm.service` and `obs_scm.service` 178 | - [`appimage.service`](https://github.com/openSUSE/obs-service-tar_scm/blob/master/appimage.service) 179 | - [`snapcraft.service`](https://github.com/openSUSE/obs-service-tar_scm/blob/master/snapcraft.service) 180 | - [`tar.service`](https://github.com/openSUSE/obs-service-tar_scm/blob/master/tar.service) 181 | - The ["Using Source Services" chapter](https://openbuildservice.org/help/manuals/obs-user-guide/cha.obs.source_service.html) 182 | of [the OBS User Guide](https://openbuildservice.org/help/manuals/obs-user-guide/) 183 | 184 | ## Test suite 185 | 186 | See the [TESTING.md](TESTING.md) file. 187 | 188 | ## Contributions 189 | 190 | See the [CONTRIBUTING.md](CONTRIBUTING.md) file. 191 | -------------------------------------------------------------------------------- /TESTING.md: -------------------------------------------------------------------------------- 1 | # `tar_scm` testing 2 | 3 | ## Unit tests 4 | 5 | Run the unit test suite via: 6 | 7 | export PYTHONPATH=. # or absolute path to repo 8 | python2 tests/test.py 9 | 10 | (If your distribution does not have `python2` in your `$PATH` then 11 | adjust the executable name accordingly.) 12 | 13 | The output may become easier to understand if you uncomment the 14 | 'failfast' option in `test.py`. This requires Python 2.7, however. 15 | You may also find that the buffered `STDOUT` from test failures gets 16 | displayed to the screen twice - once before the test failure (and 17 | corresponding stacktrace), and once after; in which case just grep for 18 | `/^FAIL: /` in the output and start reading from there. 19 | 20 | If you want to narrow the tests being run, to speed up testing during 21 | development, you can provide command-line arguments: 22 | 23 | # only run the 'plain' and 'subdir' git tests 24 | python2 tests/test.py test_plain test_subdir 25 | 26 | # run all git tests matching the regexp /subdir/ 27 | python2 tests/test.py /subdir/ 28 | 29 | Currently this filtering only applies to the git tests, but you can 30 | easily tweak `test.py` to change that. 31 | 32 | Note that for each test, a fresh `svn`/`git`/`hg`/`bzr` repository is 33 | created, and `tar_scm` is invoked one *or more* times in a faked-up 34 | OBS source service environment. Whenever `tar_scm` invokes the VCS 35 | for which its functionality is being tested, through modification of 36 | `$PATH` it actually invokes `scm-wrapper`, which logs the VCS 37 | invocation before continuing. 38 | 39 | ### Persistence between test runs 40 | 41 | The test fixtures create working directories for each test 42 | representing a (fake) check-out of a build service package, and each 43 | test invokes `tar_scm` on this working directory. The directory 44 | persists between tests to simulate real world use. 45 | 46 | Similarly, a fake `$HOME` directory is created, in which source 47 | repositories may be cached upon cloning from the (fake) upstream 48 | repository, and again, this `$HOME` directory is persisted between 49 | tests in order to simulate real world use. 50 | 51 | All these directories can be found under `tests/tmp/`. 52 | 53 | Upon a successful test run, these persisted directories are cleaned 54 | up. However, if the run fails, they are left behind for debugging. 55 | In this case, you may need to `rm -rf tests/tmp` prior to the next 56 | test run, otherwise you may get errors like `Invalid revision range` 57 | when a brand new repository history is constructed which conflicts 58 | with the previous run. 59 | 60 | ## PEP8 checking 61 | 62 | There's also a `pep8` rule for checking 63 | [PEP8](http://legacy.python.org/dev/peps/pep-0008/) compliance: 64 | 65 | make pep8 66 | 67 | ## Running all tests. 68 | 69 | You can run both sets of tests together via: 70 | 71 | make check 72 | -------------------------------------------------------------------------------- /TarSCM/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import sys 4 | import os 5 | 6 | from TarSCM.tasks import Tasks 7 | from TarSCM.helpers import Helpers 8 | from TarSCM.cli import Cli 9 | from TarSCM.archive import Tar 10 | from TarSCM.archive import ObsCpio 11 | from TarSCM.archive import Gbp 12 | from TarSCM.exceptions import OptionsError 13 | 14 | 15 | def run(): 16 | _cli = Cli() 17 | _cli.parse_args(sys.argv[1:]) 18 | 19 | if os.path.basename(sys.argv[0]) == "tar": 20 | _cli.scm = "tar" 21 | 22 | if os.path.basename(sys.argv[0]) == "obs_scm": 23 | _cli.use_obs_scm = True 24 | 25 | if os.path.basename(sys.argv[0]) == "appimage": 26 | _cli.appimage = True 27 | 28 | if os.path.basename(sys.argv[0]) == "snapcraft": 29 | _cli.snapcraft = True 30 | 31 | if os.path.basename(sys.argv[0]) == "obs_gbp": 32 | _cli.use_obs_gbp = True 33 | 34 | task_list = Tasks(_cli) 35 | 36 | task_list.generate_list() 37 | 38 | try: 39 | task_list.process_list() 40 | except OptionsError as exc: 41 | print(exc) 42 | sys.exit(1) 43 | 44 | task_list.finalize() 45 | 46 | task_list.cleanup() 47 | 48 | raise SystemExit(0) 49 | -------------------------------------------------------------------------------- /TarSCM/changes.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import logging 3 | import os 4 | import shutil 5 | import sys 6 | import tempfile 7 | import stat 8 | import io 9 | import locale 10 | 11 | from TarSCM.cli import Cli 12 | from TarSCM.config import Config 13 | 14 | 15 | class Changes(): 16 | def import_xml_parser(self): 17 | """Import the best XML parser available. Currently prefers lxml and 18 | falls back to xml.etree. 19 | 20 | There are some important differences in behaviour, which also 21 | depend on the Python version being used: 22 | 23 | | Python | 2.6 | 2.6 | 2.7 | 2.7 | 24 | |-----------+----------------+-------------+----------------+-------------| 25 | | module | lxml.etree | xml.etree | lxml.etree | xml.etree | 26 | |-----------+----------------+-------------+----------------+-------------| 27 | | empty | XMLSyntaxError | ExpatError | XMLSyntaxError | ParseError | 28 | | doc | "Document is | "no element | "Document is | "no element | 29 | | | empty" | found" | empty | found" | 30 | |-----------+----------------+-------------+----------------+-------------| 31 | | syntax | XMLSyntaxError | ExpatError | XMLSyntaxError | ParseError | 32 | | error | "invalid | "not well- | "invalid | "not well- | 33 | | | element name" | formed" | element name" | formed" | 34 | |-----------+----------------+-------------+----------------+-------------| 35 | | e.message | deprecated | deprecated | yes | yes | 36 | |-----------+----------------+-------------+----------------+-------------| 37 | | str() | yes | yes | yes | yes | 38 | |-----------+----------------+-------------+----------------+-------------| 39 | | @attr | yes | no | yes | yes | 40 | | selection | | | | | 41 | """ # noqa 42 | global ET 43 | 44 | try: 45 | # If lxml is available, we can use a parser that doesn't 46 | # destroy comments 47 | import lxml.etree as ET 48 | xml_parser = ET.XMLParser(remove_comments=False) 49 | except ImportError: 50 | import xml.etree.ElementTree as ET 51 | xml_parser = None 52 | if not hasattr(ET, 'ParseError'): 53 | try: 54 | import xml.parsers.expat 55 | except: 56 | raise RuntimeError("Couldn't load XML parser error class") 57 | 58 | return xml_parser 59 | 60 | def parse_servicedata_xml(self, srcdir): 61 | """Parses the XML in _servicedata. Returns None if the file doesn't 62 | exist or is empty, or the ElementTree on successful parsing, or 63 | raises any other exception generated by parsing. 64 | """ 65 | # Even if there's no _servicedata, we'll need the module later. 66 | xml_parser = self.import_xml_parser() 67 | 68 | servicedata_file = os.path.join(srcdir, "_servicedata") 69 | if not os.path.exists(servicedata_file): 70 | return None 71 | 72 | try: 73 | return ET.parse(servicedata_file, parser=xml_parser) 74 | except Exception as exc: 75 | # Tolerate an empty file, but any other parse error should be 76 | # made visible. 77 | if str(exc).startswith("Document is empty") or \ 78 | str(exc).startswith("no element found"): 79 | return None 80 | raise 81 | 82 | def extract_tar_scm_service(self, root, url): 83 | """Returns an object representing the 84 | element referencing the given URL. 85 | """ 86 | try: 87 | tar_scm_services = root.findall("service[@name='tar_scm']") 88 | except SyntaxError: 89 | raise RuntimeError( 90 | "Couldn't load an XML parser supporting attribute selection. " 91 | "Try installing lxml.") 92 | 93 | for service in tar_scm_services: 94 | for param in service.findall("param[@name='url']"): 95 | if param.text == url: 96 | return service 97 | 98 | def get_changesrevision(self, tar_scm_service): 99 | """Returns an object representing the 100 | element from the _servicedata file, or None, if it doesn't exist. 101 | """ 102 | params = tar_scm_service.findall("param[@name='changesrevision']") 103 | if not params: 104 | return None 105 | if len(params) > 1: 106 | raise RuntimeError('Found multiple ' 107 | 'elements in _servicedata.') 108 | return params[0] 109 | 110 | def read_changes_revision(self, url, srcdir, outdir): 111 | """ 112 | Reads the _servicedata file and returns a dictionary with 'revision' on 113 | success. As a side-effect it creates the _servicedata file if it 114 | doesn't exist. 'revision' is None in that case. 115 | """ 116 | write_servicedata = False 117 | 118 | xml_tree = self.parse_servicedata_xml(srcdir) 119 | if xml_tree is None: 120 | root = ET.fromstring("\n\n") 121 | write_servicedata = True 122 | else: 123 | root = xml_tree.getroot() 124 | 125 | service = self.extract_tar_scm_service(root, url) 126 | if service is None: 127 | service = ET.fromstring("""\ 128 | 129 | %s 130 | 131 | """ % url) 132 | root.append(service) 133 | write_servicedata = True 134 | 135 | if write_servicedata: 136 | ET.ElementTree(root).write(os.path.join(outdir, "_servicedata")) 137 | else: 138 | if not os.path.exists(os.path.join(outdir, "_servicedata")) or \ 139 | not os.path.samefile(os.path.join(srcdir, "_servicedata"), 140 | os.path.join(outdir, "_servicedata")): 141 | shutil.copy(os.path.join(srcdir, "_servicedata"), 142 | os.path.join(outdir, "_servicedata")) 143 | 144 | change_data = { 145 | 'revision': None 146 | } 147 | changesrevision_element = self.get_changesrevision(service) 148 | if changesrevision_element is not None: 149 | change_data['revision'] = changesrevision_element.text 150 | return change_data 151 | 152 | def write_changes_revision(self, url, outdir, new_revision): 153 | """Updates the changesrevision in the _servicedata file.""" 154 | logging.debug("Updating %s", os.path.join(outdir, '_servicedata')) 155 | 156 | xml_tree = self.parse_servicedata_xml(outdir) 157 | root = xml_tree.getroot() 158 | tar_scm_service = self.extract_tar_scm_service(root, url) 159 | if tar_scm_service is None: 160 | sys.exit("File _servicedata is missing tar_scm with URL '%s'" % 161 | url) 162 | 163 | changed = False 164 | element = self.get_changesrevision(tar_scm_service) 165 | if element is None: 166 | changed = True 167 | changesrevision = ET.fromstring( 168 | " %s\n" 169 | % new_revision) 170 | tar_scm_service.append(changesrevision) 171 | elif element.text != new_revision: 172 | element.text = new_revision 173 | changed = True 174 | 175 | if changed: 176 | xml_tree.write(os.path.join(outdir, "_servicedata")) 177 | 178 | def write_changes(self, changes_filename, changes, version, author): 179 | """Add changes to given *.changes file.""" 180 | if changes is None: 181 | logging.debug( 182 | "No changes found." 183 | " Skipping write_changes to %s", changes_filename) 184 | return 185 | 186 | logging.debug("Writing changes file %s", changes_filename) 187 | 188 | tmp_fp = tempfile.NamedTemporaryFile(delete=False) 189 | mode = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH 190 | os.chmod(tmp_fp.name, mode) 191 | tmp_filename = tmp_fp.name 192 | tmp_fp.close() 193 | tmp_fp = io.open(tmp_filename, 'w', encoding="UTF-8") 194 | 195 | dtime = datetime.datetime.utcnow().strftime('%a %b %d %H:%M:%S UTC %Y') 196 | 197 | text = '-' * 67 + '\n' 198 | text += "%s - %s\n" % (dtime, author) 199 | text += '\n' 200 | text += "- Update to version %s:\n" % version 201 | for line in changes: 202 | text += " * %s\n" % line 203 | text += '\n' 204 | 205 | old_fp = io.open(changes_filename, 'r', encoding='UTF-8') 206 | text += old_fp.read() 207 | old_fp.close() 208 | 209 | tmp_fp.write(text) 210 | tmp_fp.close() 211 | 212 | shutil.move(tmp_fp.name, changes_filename) 213 | 214 | def get_changesauthor(self, args): 215 | # return changesauthor if given as cli option 216 | if args.changesauthor: 217 | logging.debug("Found changesauthor in args.changesauthor='%s'", 218 | args.changesauthor) 219 | return args.changesauthor 220 | 221 | # return changesauthor if set by osc 222 | realname, mailaddr = os.getenv('VC_REALNAME'), os.getenv('VC_MAILADDR') 223 | if mailaddr and realname: 224 | logging.debug("Found VC_REALNAME='%s' and VC_MAILADDR='%s'", 225 | realname, mailaddr) 226 | return "%s <%s>" % (realname, mailaddr) 227 | 228 | if mailaddr: 229 | logging.debug("Found changesauthor in VC_MAILADDR='%s'", 230 | mailaddr) 231 | return mailaddr 232 | 233 | # return default changesauthor if running on server side 234 | if os.getenv('OBS_SERVICE_DAEMON'): 235 | logging.debug("Running in daemon mode. Using DEFAULT_AUTHOR='%s'", 236 | Cli.DEFAULT_AUTHOR) 237 | return Cli.DEFAULT_AUTHOR 238 | 239 | # exit if running locally (non server mode) and now changesauthor 240 | # could be determined 241 | raise SystemExit( 242 | """No changesauthor defined!\n""" 243 | """You can define it by:\n""" 244 | """ * configure 'email=' in ~/.config/osc/oscrc """ 245 | """in your default api section\n""" 246 | """ * configure """ 247 | """... in your _service file\n""" 248 | """ * using '--changesauthor' on the cli\n""" 249 | ) 250 | -------------------------------------------------------------------------------- /TarSCM/config.py: -------------------------------------------------------------------------------- 1 | import os 2 | import logging 3 | import re 4 | 5 | # python3 renaming of StringIO 6 | try: 7 | from StringIO import StringIO 8 | except: 9 | from io import StringIO 10 | 11 | # python3 renaming of ConfigParser 12 | try: 13 | import configparser 14 | except: 15 | import ConfigParser as configparser 16 | 17 | 18 | class Config(): 19 | # pylint: disable=too-few-public-methods 20 | def __init__( 21 | self, 22 | files=[['/etc/obs/services/tar_scm', True]] 23 | ): 24 | try: 25 | rc_file = [ 26 | os.path.join(os.environ['HOME'], '.obs', 'tar_scm'), 27 | True 28 | ] 29 | files.append(rc_file) 30 | except KeyError: 31 | pass 32 | 33 | self.configs = [] 34 | self.default_section = 'tar_scm' 35 | # We're in test-mode, so don't let any local site-wide 36 | # or per-user config impact the test suite. 37 | if os.getenv('TAR_SCM_CLEAN_ENV'): 38 | logging.info("Ignoring config files: test-mode detected") 39 | 40 | # fake a section header for configuration files 41 | for tmp in files: 42 | fname = tmp[0] 43 | self.fakeheader = tmp[1] 44 | if not os.path.isfile(fname): 45 | logging.debug("Config file not found: %s", fname) 46 | continue 47 | self.configs.append(self._init_config(fname)) 48 | 49 | # strip quotes from pathname 50 | for config in self.configs: 51 | for section in config.sections(): 52 | for opt in config.options(section): 53 | config.set( 54 | section, 55 | opt, 56 | re.sub( 57 | r'"(.*)"', 58 | r'\1', 59 | config.get(section, opt) 60 | ) 61 | ) 62 | 63 | def _init_config(self, fname): 64 | config = configparser.RawConfigParser() 65 | config.optionxform = str 66 | 67 | if self.fakeheader: 68 | logging.debug("Using fakeheader for file '%s'", fname) 69 | try: 70 | fake_header = '[' + self.default_section + ']\n' 71 | config.read_string(fake_header + open(fname, 'r').read(), 72 | source=fname) 73 | except AttributeError: 74 | tmp_fp = StringIO() 75 | tmp_fp.write('[' + self.default_section + ']\n') 76 | tmp_fp.write(open(fname, 'r').read()) 77 | tmp_fp.seek(0, os.SEEK_SET) 78 | config.readfp(tmp_fp) 79 | 80 | else: 81 | config.read(fname) 82 | 83 | return config 84 | 85 | def get(self, section, option): 86 | value = None 87 | # We're in test-mode, so don't let any local site-wide 88 | # or per-user config impact the test suite. 89 | if os.getenv('TAR_SCM_CLEAN_ENV'): 90 | return value 91 | 92 | if section is None and self.fakeheader: 93 | section = self.default_section 94 | 95 | logging.debug("SECTION: %s", section) 96 | for config in self.configs: 97 | try: 98 | value = config.get(section, option) 99 | except: 100 | pass 101 | 102 | return value 103 | -------------------------------------------------------------------------------- /TarSCM/exceptions.py: -------------------------------------------------------------------------------- 1 | class OptionsError(BaseException): 2 | pass 3 | 4 | 5 | class GitError(BaseException): 6 | pass 7 | -------------------------------------------------------------------------------- /TarSCM/helpers.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import datetime 4 | import os 5 | import logging 6 | import subprocess 7 | import io 8 | 9 | # python3 renaming of StringIO 10 | try: 11 | import StringIO 12 | except: 13 | from io import StringIO 14 | 15 | 16 | def file_write_legacy(fname, string, *args): 17 | '''function to write string to file python 2/3 compatible''' 18 | mode = 'w' 19 | if args: 20 | mode = args[0] 21 | 22 | with io.open(fname, mode, encoding='utf-8') as outfile: 23 | # 'str().encode().decode()' is required for pyhton 2/3 compatibility 24 | outfile.write(str(string).encode('UTF-8').decode('UTF-8')) 25 | 26 | 27 | class Helpers(): 28 | def run_cmd(self, cmd, cwd, interactive=False, raisesysexit=False): 29 | """ 30 | Execute the command cmd in the working directory cwd and check return 31 | value. If the command returns non-zero and raisesysexit is True raise a 32 | SystemExit exception otherwise return a tuple of return code and 33 | command output. 34 | """ 35 | logging.debug("COMMAND: %s" % cmd) 36 | 37 | proc = subprocess.Popen(cmd, 38 | shell=False, 39 | stdout=subprocess.PIPE, 40 | stderr=subprocess.STDOUT, 41 | cwd=cwd) 42 | output = '' 43 | if interactive: 44 | stdout_lines = [] 45 | while proc.poll() is None: 46 | for line in proc.stdout: 47 | line_str = line.rstrip().decode('UTF-8') 48 | print(line_str) 49 | stdout_lines.append(line_str) 50 | output = '\n'.join(stdout_lines) 51 | output = output 52 | else: 53 | output = proc.communicate()[0] 54 | if isinstance(output, bytes): 55 | output = output.decode('UTF-8') 56 | 57 | if proc.returncode and raisesysexit: 58 | logging.info("ERROR(%d): %s", proc.returncode, repr(output)) 59 | raise SystemExit( 60 | "Command %s failed(%d): '%s'" % (cmd, proc.returncode, output) 61 | ) 62 | else: 63 | logging.debug("RESULT(%d): %s", proc.returncode, repr(output)) 64 | 65 | return (proc.returncode, output) 66 | 67 | def safe_run(self, cmd, cwd, interactive=False): 68 | """Execute the command cmd in the working directory cwd and check 69 | return value. If the command returns non-zero raise a SystemExit 70 | exception. 71 | """ 72 | result = self.run_cmd(cmd, cwd, interactive, raisesysexit=True) 73 | return result 74 | 75 | def get_timestamp(self, scm_object, args, clone_dir): 76 | """Returns the commit timestamp for checked-out repository.""" 77 | 78 | timestamp = scm_object.get_timestamp() 79 | logging.debug("COMMIT TIMESTAMP: %s (%s)", timestamp, 80 | datetime.datetime.fromtimestamp(timestamp).strftime( 81 | '%Y-%m-%d %H:%M:%S')) 82 | return timestamp 83 | -------------------------------------------------------------------------------- /TarSCM/scm/__init__.py: -------------------------------------------------------------------------------- 1 | from TarSCM.scm.git import Git 2 | from TarSCM.scm.bzr import Bzr 3 | from TarSCM.scm.hg import Hg 4 | from TarSCM.scm.svn import Svn 5 | from TarSCM.scm.tar import Tar 6 | -------------------------------------------------------------------------------- /TarSCM/scm/bzr.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import re 3 | import os 4 | import dateutil.parser 5 | from TarSCM.scm.base import Scm 6 | 7 | 8 | class Bzr(Scm): 9 | scm = 'bzr' 10 | 11 | def _get_scm_cmd(self): 12 | """Compose a BZR-specific command line using http proxies.""" 13 | # Bazaar honors the http[s]_proxy variables, no action needed 14 | return [self.scm] 15 | 16 | def fetch_upstream_scm(self): 17 | """SCM specific version of fetch_uptream for bzr.""" 18 | 19 | self.auth_url() 20 | command = self._get_scm_cmd() + ['checkout', self.url, self.clone_dir] 21 | if self.revision: 22 | command.insert(3, '-r') 23 | command.insert(4, self.revision) 24 | if not self.is_sslverify_enabled(): 25 | command.insert(2, '-Ossl.cert_reqs=None') 26 | wdir = os.path.abspath(os.path.join(self.clone_dir, os.pardir)) 27 | self.helpers.safe_run(command, wdir, interactive=sys.stdout.isatty()) 28 | 29 | def update_cache(self): 30 | """Update sources via bzr.""" 31 | # pylint: disable=duplicate-code 32 | command = self._get_scm_cmd() + ['update'] 33 | if self.revision: 34 | command.insert(3, '-r') 35 | command.insert(4, self.revision) 36 | 37 | self.helpers.safe_run( 38 | command, 39 | cwd=self.clone_dir, 40 | interactive=sys.stdout.isatty() 41 | ) 42 | 43 | def detect_version(self, args): 44 | """ 45 | Automatic detection of version number for checked-out BZR repository. 46 | """ 47 | versionformat = args['versionformat'] 48 | if versionformat is None: 49 | versionformat = '%r' 50 | 51 | version = self.helpers.safe_run(self._get_scm_cmd() + ['revno'], 52 | self.clone_dir)[1] 53 | return re.sub('%r', version.strip(), versionformat) 54 | 55 | def get_timestamp(self): 56 | log = self.helpers.safe_run( 57 | self._get_scm_cmd() + ['log', '--limit=1', '--log-format=long'], 58 | self.clone_dir 59 | )[1] 60 | match = re.search(r'timestamp:(.*)', log, re.MULTILINE) 61 | if not match: 62 | return 0 63 | tsm = match.group(1).strip() 64 | timestamp = dateutil.parser.parse(tsm).strftime("%s") 65 | return int(timestamp) 66 | 67 | # no cleanup is necessary for bzr 68 | def cleanup(self): 69 | pass 70 | 71 | def check_url(self): 72 | """check if url is a remote url""" 73 | if not re.match("^((a?ftp|bzr|https?)://|lp:)", self.url): 74 | return False 75 | return True 76 | -------------------------------------------------------------------------------- /TarSCM/scm/hg.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import re 3 | import os 4 | import tempfile 5 | import shutil 6 | import logging 7 | from TarSCM.scm.base import Scm 8 | 9 | 10 | class Hg(Scm): 11 | scm = 'hg' 12 | 13 | hgtmpdir = tempfile.mkdtemp() 14 | 15 | def _get_scm_cmd(self): 16 | """Compose a HG-specific command line using http proxies.""" 17 | # Mercurial requires declaring proxies via a --config parameter 18 | scmcmd = ['hg'] 19 | if self.httpproxy: 20 | logging.debug("using tempdir: %s", self.hgtmpdir) 21 | cfg = open(self.hgtmpdir + "/tempsettings.rc", "wb") 22 | cfg.write('[http_proxy]\n') 23 | 24 | regexp_proxy = re.match('http://(.*):(.*)', 25 | self.httpproxy, 26 | re.M | re.I) 27 | 28 | proxy_host = regexp_proxy.group(1) 29 | proxy_port = regexp_proxy.group(2) 30 | 31 | if proxy_host is not None: 32 | logging.debug('using proxy host: %s', proxy_host) 33 | cfg.write('host=' + proxy_host) 34 | if proxy_port is not None: 35 | logging.debug('using proxy port: %s', proxy_port) 36 | cfg.write('port=' + proxy_port) 37 | if self.noproxy is not None: 38 | logging.debug('using proxy exceptions: %s', self.noproxy) 39 | cfg.write('no=' + self.noproxy) 40 | cfg.close() 41 | 42 | # we just point Mercurial to where the config file is 43 | os.environ['HGRCPATH'] = self.hgtmpdir 44 | 45 | return scmcmd 46 | 47 | def switch_revision(self): 48 | """Switch sources to revision.""" 49 | if self.revision is None: 50 | self.revision = 'tip' 51 | 52 | cmd = self._get_scm_cmd() + ['update', self.revision] 53 | rcode, _ = self.helpers.run_cmd(cmd, 54 | cwd=self.clone_dir, 55 | interactive=sys.stdout.isatty()) 56 | if rcode: 57 | sys.exit('%s: No such revision' % self.revision) 58 | 59 | def fetch_upstream_scm(self): 60 | """SCM specific version of fetch_uptream for hg.""" 61 | self.auth_url() 62 | command = self._get_scm_cmd() + ['clone', self.url, self.clone_dir] 63 | if not self.is_sslverify_enabled(): 64 | command += ['--insecure'] 65 | wdir = os.path.abspath(os.path.join(self.clone_dir, os.pardir)) 66 | self.helpers.safe_run(command, wdir, 67 | interactive=sys.stdout.isatty()) 68 | 69 | def update_cache(self): 70 | """Update sources via hg.""" 71 | try: 72 | self.helpers.safe_run(self._get_scm_cmd() + ['pull'], 73 | cwd=self.clone_dir, 74 | interactive=sys.stdout.isatty()) 75 | except SystemExit as exc: 76 | # Contrary to the docs, hg pull returns exit code 1 when 77 | # there are no changes to pull, but we don't want to treat 78 | # this as an error. 79 | if re.match('.*no changes found.*', str(exc)) is None: 80 | raise 81 | 82 | def detect_version(self, args): 83 | """ 84 | Automatic detection of version number for checked-out HG repository. 85 | """ 86 | versionformat = args['versionformat'] 87 | if versionformat is None: 88 | versionformat = '{rev}' 89 | 90 | cmd = self._get_scm_cmd() + ['id', '-n'] 91 | version = self.helpers.safe_run(cmd, self.clone_dir)[1] 92 | 93 | # Mercurial internally stores commit dates in its changelog 94 | # context objects as (epoch_secs, tz_delta_to_utc) tuples (see 95 | # mercurial/util.py). For example, if the commit was created 96 | # whilst the timezone was BST (+0100) then tz_delta_to_utc is 97 | # -3600. In this case, 98 | # 99 | # hg log -l1 -r$rev --template '{date}\n' 100 | # 101 | # will result in something like '1375437706.0-3600' where the 102 | # first number is timezone-agnostic. However, hyphens are not 103 | # permitted in rpm version numbers, so tar_scm removes them via 104 | # sed. This is required for this template format for any time 105 | # zone "numerically east" of UTC. 106 | # 107 | # N.B. since the extraction of the timestamp as a version number 108 | # is generally done in order to provide chronological sorting, 109 | # ideally we would ditch the second number. However the 110 | # template format string is left up to the author of the 111 | # _service file, so we can't do it here because we don't know 112 | # what it will expand to. Mercurial provides template filters 113 | # for dates (e.g. 'hgdate') which _service authors could 114 | # potentially use, but unfortunately none of them can easily 115 | # extract only the first value from the tuple, except for maybe 116 | # 'sub(...)' which is only available since 2.4 (first introduced 117 | # in openSUSE 12.3). 118 | 119 | cmd = self._get_scm_cmd() 120 | cmd.extend([ 121 | 'log', 122 | '-l1', 123 | "-r%s" % version.strip(), 124 | '--template', 125 | versionformat 126 | ]) 127 | 128 | return self.helpers.safe_run(cmd, self.clone_dir)[1] 129 | 130 | def get_timestamp(self): 131 | data = {"parent_tag": None, "versionformat": "{date}"} 132 | timestamp = self.detect_version(data) 133 | timestamp = re.sub(r'([0-9]+)\..*', r'\1', timestamp) 134 | return int(timestamp) 135 | 136 | def cleanup(self): 137 | try: 138 | shutil.rmtree(self.hgtmpdir, ignore_errors=True) 139 | except: 140 | logging.debug("error on cleanup: %s", sys.exc_info()[0]) 141 | raise 142 | 143 | def check_url(self): 144 | """check if url is a remote url""" 145 | if not re.match("^https?://", self.url): 146 | return False 147 | return True 148 | -------------------------------------------------------------------------------- /TarSCM/scm/svn.py: -------------------------------------------------------------------------------- 1 | import hashlib 2 | import sys 3 | import re 4 | import os 5 | import logging 6 | import tempfile 7 | import shutil 8 | 9 | import dateutil.parser 10 | 11 | from TarSCM.scm.base import Scm 12 | 13 | ENCODING_RE = re.compile(r".*(" 14 | "Can't convert string from '.*' to native encoding:" 15 | "|" 16 | "Can't convert string from native encoding to ).*") 17 | 18 | ENCODING_MSG = ("Encoding error! " 19 | "Please specify proper --locale parameter " 20 | "or '...' " 21 | "in your service file!") 22 | 23 | 24 | class Svn(Scm): 25 | scm = 'svn' 26 | 27 | svntmpdir = tempfile.mkdtemp() 28 | 29 | def _get_scm_cmd(self): 30 | """Compose a SVN-specific command line using http proxies.""" 31 | # Subversion requires declaring proxies in a file, as it does not 32 | # support the http[s]_proxy variables. This creates the temporary 33 | # config directory that will be added via '--config-dir' 34 | scmcmd = ['svn'] 35 | if self.httpproxy: 36 | logging.debug("using svntmpdir %s", self.svntmpdir) 37 | cfg = open(self.svntmpdir + "/servers", "wb") 38 | cfg.write('[global]\n') 39 | 40 | re_proxy = re.match('http://(.*):(.*)', 41 | self.httpproxy, 42 | re.M | re.I) 43 | 44 | proxy_host = re_proxy.group(1) 45 | proxy_port = re_proxy.group(2) 46 | 47 | if proxy_host is not None: 48 | logging.debug('using proxy host: %s', proxy_host) 49 | cfg.write('http-proxy-host=' + proxy_host + '\n') 50 | 51 | if proxy_port is not None: 52 | logging.debug('using proxy port: %s', proxy_port) 53 | cfg.write('http-proxy-port=' + proxy_port + '\n') 54 | 55 | if self.noproxy is not None: 56 | logging.debug('using proxy exceptions: %s', self.noproxy) 57 | no_proxy_domains = [] 58 | no_proxy_domains.append(tuple(self.noproxy.split(","))) 59 | no_proxy_string = "" 60 | 61 | # for some odd reason subversion expects the domains 62 | # to have an asterisk 63 | for i in range(len(no_proxy_domains[0])): 64 | tmpstr = str(no_proxy_domains[0][i]).strip() 65 | if tmpstr.startswith('.'): 66 | no_proxy_string += '*' + tmpstr 67 | else: 68 | no_proxy_string += tmpstr 69 | 70 | if i < len(no_proxy_domains[0]) - 1: 71 | no_proxy_string += ',' 72 | 73 | no_proxy_string += '\n' 74 | logging.debug('no_proxy string = %s', no_proxy_string) 75 | cfg.write('http-proxy-exceptions=' + no_proxy_string) 76 | cfg.close() 77 | scmcmd += ['--config-dir', self.svntmpdir] 78 | 79 | if self.user and self.password: 80 | scmcmd += ['--username', self.user] 81 | scmcmd += ['--password', self.password] 82 | 83 | return scmcmd 84 | 85 | def fetch_upstream_scm(self): 86 | """SCM specific version of fetch_uptream for svn.""" 87 | command = self._get_scm_cmd() + ['checkout', '--non-interactive', 88 | self.url, self.clone_dir] 89 | if self.revision: 90 | command.insert(4, '-r%s' % self.revision) 91 | if not self.is_sslverify_enabled(): 92 | command.insert(3, '--trust-server-cert') 93 | 94 | wdir = os.path.abspath(os.path.join(self.clone_dir, os.pardir)) 95 | 96 | try: 97 | self.helpers.safe_run(command, wdir, 98 | interactive=sys.stdout.isatty()) 99 | except SystemExit as exc: 100 | if re.search(ENCODING_RE, exc.code): 101 | raise SystemExit(ENCODING_MSG) # pylint: disable=E0012,W0707 102 | raise exc 103 | 104 | def update_cache(self): 105 | """Update sources via svn.""" 106 | command = self._get_scm_cmd() + ['update'] 107 | if self.revision: 108 | command.insert(3, "-r%s" % self.revision) 109 | 110 | try: 111 | self.helpers.safe_run(command, cwd=self.clone_dir, 112 | interactive=sys.stdout.isatty()) 113 | except SystemExit as exc: 114 | logging.warning("Could not update cache: >>>%s<<.*?", xml_lines, re.S) 211 | 212 | for line in lines: 213 | line = line.replace("", "").replace("", "") 214 | new_lines = new_lines + line.split("\n") 215 | 216 | return new_lines 217 | 218 | def _get_rev(self, clone_dir, num_commits): 219 | cmd = self._get_scm_cmd() 220 | cmd.extend(['log', '-l%d' % num_commits, '-q', '--incremental']) 221 | raw = self.helpers.safe_run(cmd, cwd=clone_dir) 222 | revisions = raw[1].split("\n") 223 | # remove blank entry on end 224 | revisions.pop() 225 | # return last entry 226 | revision = revisions[-1] 227 | # retrieve the revision number and remove r 228 | revision = re.search(r'^r[0-9]*', revision, re.M) 229 | revision = revision.group().replace("r", "") 230 | return revision 231 | 232 | def cleanup(self): 233 | try: 234 | shutil.rmtree(self.svntmpdir, ignore_errors=True) 235 | except: 236 | logging.debug("error on cleanup: %s", sys.exc_info()[0]) 237 | raise 238 | 239 | def check_url(self): 240 | """check if url is a remote url""" 241 | if not re.match("^(https?|svn)://", self.url): 242 | return False 243 | return True 244 | -------------------------------------------------------------------------------- /TarSCM/scm/tar.py: -------------------------------------------------------------------------------- 1 | import glob 2 | import os 3 | import sys 4 | 5 | from TarSCM.scm.base import Scm 6 | 7 | 8 | class Tar(Scm): 9 | scm = 'tar' 10 | 11 | def fetch_upstream(self): 12 | """SCM specific version of fetch_upstream for tar.""" 13 | version = None 14 | if self.args.obsinfo: 15 | self.basename = self.clone_dir = self.read_from_obsinfo( 16 | self.args.obsinfo, "name" 17 | ) 18 | version = self.read_from_obsinfo(self.args.obsinfo, "version") 19 | 20 | if self.args.filename: 21 | self.basename = self.clone_dir = self.args.filename 22 | 23 | if self.args.version and self.args.version != '_auto_': 24 | version = self.args.version 25 | 26 | if not self.basename or not self.clone_dir: 27 | raise SystemExit("ERROR: no .obsinfo file found in directory\n" 28 | " and no manual configuration: " 29 | "'%s'" % os.getcwd()) 30 | if "/" in self.clone_dir: 31 | sys.exit("name in obsinfo contains '/'.") 32 | 33 | if "/" in version or '..' in version: 34 | raise SystemExit("version in obsinfo contains '/' or '..'.") 35 | 36 | if version != '' and version != '_none_': 37 | self.clone_dir += "-" + version 38 | 39 | if not os.path.exists(self.clone_dir) \ 40 | and self.basename != self.clone_dir: 41 | self._final_rename_needed = True 42 | # not need in case of local osc build 43 | try: 44 | os.rename(self.basename, self.clone_dir) 45 | except OSError: 46 | raise SystemExit( 47 | "Error while moving from '%s' to '%s'\n" 48 | "Current working directory: '%s'" % 49 | (self.basename, self.clone_dir, os.getcwd()) 50 | ) 51 | 52 | def update_cache(self): 53 | """Update sources via tar.""" 54 | pass 55 | 56 | def detect_version(self, args): 57 | """Read former stored version.""" 58 | if self.args.obsinfo: 59 | return self.read_from_obsinfo(self.args.obsinfo, "version") 60 | 61 | def get_timestamp(self): 62 | if self.args.obsinfo: 63 | return int(self.read_from_obsinfo(self.args.obsinfo, "mtime")) 64 | if self.args.filename: 65 | return int(os.path.getmtime(self.args.filename)) 66 | 67 | def read_from_obsinfo(self, filename, key): 68 | infofile = open(filename, "r") 69 | line = infofile.readline() 70 | while line: 71 | k = line.split(":", 1) 72 | if k[0] == key: 73 | return k[1].strip() 74 | line = infofile.readline() 75 | return "" 76 | 77 | def finalize(self): 78 | """Execute final cleanup of workspace""" 79 | if self._final_rename_needed: 80 | os.rename(self.clone_dir, self.basename) 81 | 82 | # no cleanup is necessary for tar 83 | def cleanup(self): 84 | pass 85 | -------------------------------------------------------------------------------- /TarSCM/tasks.py: -------------------------------------------------------------------------------- 1 | ''' 2 | This module contains the class tasks 3 | ''' 4 | from __future__ import print_function 5 | 6 | import glob 7 | import copy 8 | import atexit 9 | import logging 10 | import os 11 | import shutil 12 | import sys 13 | import re 14 | import locale 15 | import json 16 | import io 17 | import yaml 18 | 19 | import TarSCM.scm 20 | import TarSCM.archive 21 | from TarSCM.helpers import Helpers, file_write_legacy 22 | from TarSCM.changes import Changes 23 | from TarSCM.exceptions import OptionsError 24 | 25 | 26 | class Tasks(): 27 | # pylint: disable=too-many-branches 28 | ''' 29 | Class to create a task list for formats which can contain more then one scm 30 | job like snapcraft or appimage 31 | ''' 32 | def __init__(self, args): 33 | self.task_list = [] 34 | self.cleanup_dirs = [] 35 | self.helpers = Helpers() 36 | self.changes = Changes() 37 | self.scm_object = None 38 | self.data_map = None 39 | self.args = args 40 | 41 | def cleanup(self): 42 | """Cleaning temporary directories.""" 43 | if self.args.skip_cleanup: 44 | logging.debug("Skipping cleanup") 45 | return 46 | 47 | logging.debug("Cleaning: %s", ' '.join(self.cleanup_dirs)) 48 | 49 | for dirname in self.cleanup_dirs: 50 | if not os.path.exists(dirname): 51 | continue 52 | ploc = locale.getpreferredencoding() 53 | shutil.rmtree(dirname.encode(ploc)) 54 | self.cleanup_dirs = [] 55 | # Unlock to prevent dead lock in cachedir if exception 56 | # gets raised 57 | if self.scm_object: 58 | self.scm_object.unlock_cache() 59 | # calls the corresponding cleanup routine 60 | self.scm_object.cleanup() 61 | 62 | def generate_list(self): 63 | ''' 64 | Generate list of scm jobs from appimage.yml, snapcraft.yaml or a single 65 | job from cli arguments. 66 | ''' 67 | args = self.args 68 | scms = ['git', 'tar', 'svn', 'bzr', 'hg'] 69 | 70 | if args.appimage: 71 | # we read the SCM config from appimage.yml 72 | with io.open('appimage.yml', encoding='utf-8') as filehandle: 73 | self.data_map = yaml.safe_load(filehandle) 74 | args.use_obs_scm = True 75 | build_scms = () 76 | try: 77 | build_scms = self.data_map['build'].keys() 78 | except (TypeError, KeyError): 79 | pass 80 | # run for each scm an own task 81 | for scm in scms: 82 | if scm not in build_scms: 83 | continue 84 | for url in self.data_map['build'][scm]: 85 | args.url = url 86 | args.scm = scm 87 | self.task_list.append(copy.copy(args)) 88 | 89 | elif args.snapcraft: 90 | # we read the SCM config from snapcraft.yaml instead 91 | # getting it via parameters 92 | with io.open('snapcraft.yaml', encoding='utf-8') as filehandle: 93 | self.data_map = yaml.safe_load(filehandle) 94 | args.use_obs_scm = True 95 | # run for each part an own task 96 | for part in self.data_map['parts'].keys(): 97 | args.filename = part 98 | if 'source-type' not in self.data_map['parts'][part].keys(): 99 | continue 100 | pep8_1 = self.data_map['parts'][part]['source-type'] 101 | if pep8_1 not in scms: 102 | continue 103 | # avoid conflicts with files 104 | args.clone_prefix = "_obs_" 105 | args.url = self.data_map['parts'][part]['source'] 106 | self.data_map['parts'][part]['source'] = part 107 | args.scm = self.data_map['parts'][part]['source-type'] 108 | del self.data_map['parts'][part]['source-type'] 109 | self.task_list.append(copy.copy(args)) 110 | 111 | # only try to autodetect obsinfo files if no obsinfo file is given 112 | # as cli parameter 113 | elif args.scm == 'tar' and args.obsinfo is None: 114 | # use all obsinfo files in cwd if none are given 115 | files = glob.glob('*.obsinfo') 116 | if files: 117 | for obsinfo in files: 118 | if obsinfo != '_scmsync.obsinfo': 119 | args.obsinfo = obsinfo 120 | self.task_list.append(copy.copy(args)) 121 | if not args.obsinfo: 122 | # Fallback if there are no obsinfo files 123 | self.task_list.append(args) 124 | else: 125 | self.task_list.append(args) 126 | 127 | def process_list(self): 128 | ''' 129 | process tasks from the task_list 130 | ''' 131 | for task in self.task_list: 132 | self.process_single_task(task) 133 | 134 | def finalize(self): 135 | ''' 136 | final steps after processing task list 137 | ''' 138 | args = self.args 139 | if args.snapcraft: 140 | # write the new snapcraft.yaml file 141 | # we prefix our own here to be sure to not overwrite user files, 142 | # if they are using it in "disabled" mode 143 | new_file = args.outdir + '/_service:snapcraft:snapcraft.yaml' 144 | yml_str = yaml.dump(self.data_map, default_flow_style=False) 145 | file_write_legacy(new_file, yml_str) 146 | 147 | # execute also download_files for downloading single sources 148 | if args.snapcraft or args.appimage: 149 | download_files = '/usr/lib/obs/service/download_files' 150 | if os.path.exists(download_files): 151 | cmd = [download_files, '--outdir', args.outdir] 152 | rcode, output = self.helpers.run_cmd(cmd, None) 153 | 154 | if rcode != 0: 155 | raise RuntimeError("download_files has failed:%s" % output) 156 | 157 | def check_for_branch_request(self): 158 | # we may have a _branch_request file. In that case we life in a 159 | # branch create by a webhook from github or gitlab pull/merge request 160 | # the source supposed to be merged is more important then the code 161 | # referenced in the _service file. 162 | args = self.args 163 | if not os.path.exists('_branch_request'): 164 | return args 165 | 166 | # is it a branch request? 167 | with io.open("_branch_request", "r", encoding='utf-8') as ofh: 168 | dat = json.load(ofh) 169 | if dat.get('object_kind') == 'merge_request': 170 | # gitlab merge request 171 | args.url = dat['project']['http_url'] 172 | rev = dat['object_attributes']['source']['default_branch'] 173 | args.revision = rev 174 | elif dat.get('action') == 'opened': 175 | # github pull request 176 | args.url = "https://github.com/" 177 | args.url += dat['pull_request']['head']['repo']['full_name'] 178 | args.revision = dat['pull_request']['head']['sha'] 179 | 180 | return args 181 | 182 | def process_single_task(self, args): 183 | ''' 184 | do the work for a single task 185 | ''' 186 | 187 | self.args = self.check_for_branch_request() 188 | 189 | logging.basicConfig(format="%(message)s", stream=sys.stderr, 190 | level=logging.INFO) 191 | 192 | # force cleaning of our workspace on exit 193 | atexit.register(self.cleanup) 194 | 195 | scm2class = { 196 | 'git': 'Git', 197 | 'bzr': 'Bzr', 198 | 'hg': 'Hg', 199 | 'svn': 'Svn', 200 | 'tar': 'Tar', 201 | } 202 | 203 | # create objects for TarSCM. and TarSCM.helpers 204 | try: 205 | scm_class = getattr(TarSCM.scm, scm2class[args.scm]) 206 | except: 207 | msg = "Please specify valid --scm=... options" 208 | raise OptionsError(msg) # pylint: disable=E0012,W0707 209 | 210 | # self.scm_object is need to unlock cache in cleanup 211 | # if exception occurs 212 | self.scm_object = scm_object = scm_class(args, self) 213 | 214 | tmode = bool(os.getenv('TAR_SCM_TESTMODE')) 215 | if not tmode and not scm_object.check_url(): 216 | sys.exit("--url does not match remote repository") 217 | 218 | try: 219 | scm_object.check_scm() 220 | except OSError: 221 | print("Please install '%s'" % scm_object.scm) 222 | sys.exit(1) 223 | 224 | scm_object.fetch_upstream() 225 | version = self.get_version() 226 | 227 | (dstname, changesversion, basename) = self._dstname(scm_object, 228 | version) 229 | 230 | logging.debug("DST: %s", dstname) 231 | 232 | detected_changes = scm_object.detect_changes() 233 | 234 | if not args.use_obs_gbp: 235 | scm_object.prep_tree_for_archive(args.subdir, args.outdir, 236 | dstname=dstname) 237 | self.cleanup_dirs.append(scm_object.arch_dir) 238 | 239 | # For the GBP service there is no copy in arch_dir, so use clone_dir 240 | # which has the same content 241 | extract_src = scm_object.arch_dir 242 | if args.use_obs_scm: 243 | arch = TarSCM.archive.ObsCpio() 244 | elif args.use_obs_gbp: 245 | arch = TarSCM.archive.Gbp() 246 | extract_src = scm_object.clone_dir 247 | else: 248 | arch = TarSCM.archive.Tar() 249 | 250 | arch.extract_from_archive(extract_src, args.extract, 251 | args.outdir) 252 | 253 | arch.create_archive( 254 | scm_object, 255 | basename = basename, 256 | dstname = dstname, 257 | version = version, 258 | cli = args 259 | ) 260 | 261 | if detected_changes: 262 | self._process_changes(args, 263 | version, 264 | changesversion, 265 | detected_changes) 266 | 267 | scm_object.finalize() 268 | 269 | def _dstname(self, scm_object, version): 270 | args = self.args 271 | if args.filename: 272 | basename = dstname = args.filename 273 | else: 274 | basename = dstname = os.path.basename(scm_object.clone_dir) 275 | 276 | if version and not sys.argv[0].endswith("/tar") \ 277 | and not sys.argv[0].endswith("/snapcraft") \ 278 | and not sys.argv[0].endswith("/appimage"): 279 | if isinstance(dstname, bytes): 280 | version = version.encode('UTF-8') 281 | if not args.without_version: 282 | dstname += '-' + version 283 | return (dstname, version, basename) 284 | 285 | def _process_changes(self, args, ver, changesversion, detected_changes): 286 | changesauthor = self.changes.get_changesauthor(args) 287 | 288 | logging.debug("AUTHOR: %s", changesauthor) 289 | 290 | if not ver: 291 | args.version = "_auto_" 292 | changesversion = self.get_version() 293 | 294 | logging.debug("Searching for '*.changes' in %s", os.getcwd()) 295 | for filename in glob.glob('*.changes'): 296 | new_changes_file = os.path.join(args.outdir, filename) 297 | shutil.copy(filename, new_changes_file) 298 | self.changes.write_changes(new_changes_file, 299 | detected_changes['lines'], 300 | changesversion, changesauthor) 301 | self.changes.write_changes_revision(args.url, args.outdir, 302 | detected_changes['revision']) 303 | 304 | def get_version(self): 305 | ''' 306 | Generate final version number by detecting version from scm if not 307 | given as cli option and applying versionrewrite_pattern and 308 | versionprefix if given as cli option 309 | ''' 310 | version = self.args.version 311 | if version == '_none_': 312 | return '' 313 | if version == '_auto_' or self.args.versionformat: 314 | version = self.detect_version() 315 | if self.args.versionrewrite_pattern: 316 | regex = re.compile(self.args.versionrewrite_pattern) 317 | version = regex.sub(self.args.versionrewrite_replacement, version) 318 | else: 319 | args = self.args.__dict__ 320 | debian = args.get('use_obs_gbp', False) 321 | version = self.scm_object.version_iso_cleanup(version, debian) 322 | if self.args.versionprefix: 323 | version = "%s.%s" % (self.args.versionprefix, version) 324 | 325 | logging.debug("VERSION(auto): %s", version) 326 | return version 327 | 328 | def detect_version(self): 329 | """Automatic detection of version number for checked-out repository.""" 330 | 331 | version = self.scm_object.detect_version(self.args.__dict__).strip() 332 | logging.debug("VERSION(auto): %s", version) 333 | return version 334 | -------------------------------------------------------------------------------- /appimage: -------------------------------------------------------------------------------- 1 | tar_scm.py -------------------------------------------------------------------------------- /appimage.service: -------------------------------------------------------------------------------- 1 | 2 | handle sources specified in appimage.yml 3 | This service needs to be executed to download sources according to appimage.yml file 4 | 5 | Specify whether to include git submodules. Default is 'enable'. main or master is override the specified commit with master or main branch. 6 | enable 7 | master 8 | main 9 | disable 10 | 11 | 12 | Specify whether to include git lfs blobs. Default is 'disable'. 13 | enable 14 | disable 15 | 16 | 17 | Specify Whether or not to check server certificate against installed CAs. Default is 'enable'. 18 | enable 19 | disable 20 | 21 | 22 | -------------------------------------------------------------------------------- /debian/compat: -------------------------------------------------------------------------------- 1 | 10 2 | -------------------------------------------------------------------------------- /debian/control: -------------------------------------------------------------------------------- 1 | Source: obs-service-tar-scm 2 | Section: devel 3 | Priority: optional 4 | Maintainer: Daniel Gollub 5 | Build-Depends: debhelper (>= 8.0.0), python3, python3-dateutil, dh-python 6 | Standards-Version: 3.9.3 7 | Homepage: https://github.com/openSUSE/obs-service-tar_scm 8 | 9 | Package: obs-service-tar-scm 10 | Architecture: all 11 | Provides: obs-service-obs-scm, obs-service-tar, obs-service-gbp 12 | Depends: ${misc:Depends}, ${python3:Depends}, python3, bzr, git, subversion, cpio, python3-dateutil, python3-yaml, locales-all 13 | Recommends: mercurial, git-buildpackage, git-lfs 14 | Description: OBS source service: fetches SCM tarballs 15 | This is a source service for openSUSE Build Service. 16 | It supports downloading from svn, git, hg and bzr repositories. 17 | -------------------------------------------------------------------------------- /debian/copyright: -------------------------------------------------------------------------------- 1 | Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ 2 | Upstream-Name: obs-service-tar_scm 3 | Source: https://github.com/openSUSE/obs-service-tar_scm 4 | 5 | Files: * 6 | Copyright: 2010 by Adrian Schröter 7 | License: GPL-2.0+ 8 | 9 | Files: debian/* 10 | Copyright: 2014 Daniel Gollub 11 | License: GPL-2.0+ 12 | 13 | License: GPL-2.0+ 14 | This package is free software; you can redistribute it and/or modify 15 | it under the terms of the GNU General Public License as published by 16 | the Free Software Foundation; either version 2 of the License, or 17 | (at your option) any later version. 18 | . 19 | This package is distributed in the hope that it will be useful, 20 | but WITHOUT ANY WARRANTY; without even the implied warranty of 21 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 22 | GNU General Public License for more details. 23 | . 24 | You should have received a copy of the GNU General Public License 25 | along with this program. If not, see 26 | . 27 | On Debian systems, the complete text of the GNU General 28 | Public License version 2 can be found in "/usr/share/common-licenses/GPL-2". 29 | -------------------------------------------------------------------------------- /debian/rules: -------------------------------------------------------------------------------- 1 | #!/usr/bin/make -f 2 | # -*- makefile -*- 3 | # 4 | export DH_VERBOSE=1 5 | DESTDIR=debian/obs-service-tar-scm 6 | 7 | %: 8 | dh $@ --with python3 9 | 10 | # Skip the Makefile and just rely on the python debhelper 11 | override_dh_auto_build: 12 | 13 | # Skip tests as slow and don't detect python on debian 14 | override_dh_auto_test: 15 | 16 | override_dh_auto_install: 17 | dh_auto_install -- COMPILED_PYTHON=false WITH_GBP=1 install 18 | -------------------------------------------------------------------------------- /debian/source/format: -------------------------------------------------------------------------------- 1 | 3.0 (native) 2 | -------------------------------------------------------------------------------- /dist/debian.dsc: -------------------------------------------------------------------------------- 1 | Format: 1.0 2 | Source: obs-service-tar-scm 3 | Version: 0.10.43 4 | Provides: obs-service-obs_scm, obs-service-tar, obs-service-gbp 5 | Binary: obs-service-tar_scm 6 | Maintainer: Adrian Schroeter 7 | Architecture: all 8 | Standards-Version: 3.9.3 9 | Build-Depends: debhelper (>= 8.0.0), python3, python3-dateutil, dh-python, python3-yaml 10 | 11 | Package: obs-service-tar-scm 12 | Architecture: all 13 | Provides: obs-service-obs-scm, obs-service-tar 14 | Depends: ${misc:Depends}, ${python3:Depends}, python3, bzr, git, subversion, cpio, python3-dateutil, python3-yaml 15 | Recommends: mercurial, git-buildpackage, git-lfs 16 | Description: An OBS source service: fetches SCM tarballs 17 | This is a source service for openSUSE Build Service. 18 | It supports downloading from svn, git, hg and bzr repositories. 19 | 20 | -------------------------------------------------------------------------------- /dist/obs-service-tar_scm.spec: -------------------------------------------------------------------------------- 1 | # 2 | # spec file for package obs-service-tar_scm 3 | # 4 | # Copyright (c) 2025 SUSE LLC 5 | # 6 | # All modifications and additions to the file contributed by third parties 7 | # remain the property of their copyright owners, unless otherwise agreed 8 | # upon. The license for this file, and modifications and additions to the 9 | # file, is the same license as for the pristine package itself (unless the 10 | # license for the pristine package is not an Open Source License, in which 11 | # case the license is the MIT License). An "Open Source License" is a 12 | # license that conforms to the Open Source Definition (Version 1.9) 13 | # published by the Open Source Initiative. 14 | 15 | # Please submit bugfixes or comments via https://bugs.opensuse.org/ 16 | # 17 | 18 | 19 | %if 0%{?fedora_version}%{?rhel} 20 | %define _pkg_base %nil 21 | %else 22 | %define _pkg_base -base 23 | %endif 24 | 25 | %define flavor @BUILD_FLAVOR@%nil 26 | %if "%{flavor}" == "" 27 | %define nsuffix %nil 28 | %else 29 | %define nsuffix -test 30 | %endif 31 | 32 | %if 0%{?suse_version} && 0%{?suse_version} >= 1220 && "%{flavor}" == "test" 33 | %bcond_without obs_scm_testsuite 34 | %else 35 | %bcond_with obs_scm_testsuite 36 | %endif 37 | 38 | # special guard for flavor test, yet --without test being specified 39 | %if "%{flavor}" == "test" && %{without obs_scm_testsuite} 40 | ExclusiveArch: skip-build 41 | %endif 42 | 43 | %if 0%{?suse_version} >= 1315 || 0%{?fedora_version} >= 29 || 0%{?rhel} >= 8 44 | %bcond_without python3 45 | %else 46 | %bcond_with python3 47 | %endif 48 | 49 | # This list probably needs to be extended 50 | # logic seems to be if python < 2.7 ; then needs_external_argparse ; fi 51 | %if (0%{?centos_version} == 6) || (0%{?suse_version} && 0%{?suse_version} < 1315) || (0%{?fedora_version} && 0%{?fedora_version} < 26) 52 | %bcond_without needs_external_argparse 53 | %else 54 | %bcond_with needs_external_argparse 55 | %endif 56 | 57 | %if %{with python3} 58 | %if 0%{?suse_version} > 1500 59 | %define use_python %{primary_python} 60 | %else 61 | %define use_python python3 62 | %endif 63 | %define use_test test3 64 | %else 65 | %define use_python python 66 | %define use_test test 67 | %endif 68 | 69 | %if 0%{?suse_version} 70 | %define pyyaml_package %{use_python}-PyYAML 71 | %if 0%{?suse_version} >= 1550 || 0%{?sle_version} >= 150100 72 | %define locale_package glibc-locale-base 73 | %else 74 | %define locale_package glibc-locale 75 | %endif 76 | %endif 77 | 78 | %if 0%{?fedora_version} || 0%{?rhel} 79 | %if 0%{?fedora_version} >= 29 || 0%{?rhel} >= 8 80 | %define pyyaml_package %{use_python}-PyYAML 81 | %else 82 | %define pyyaml_package PyYAML 83 | %endif 84 | 85 | %if 0%{?fedora_version} >= 24 || 0%{?rhel} >= 8 86 | %define locale_package glibc-langpack-en 87 | %else 88 | %define locale_package glibc-common 89 | %endif 90 | %endif 91 | 92 | %if 0%{?mageia} || 0%{?mandriva_version} 93 | %define pyyaml_package python-yaml 94 | %define locale_package locales 95 | %endif 96 | 97 | # Mageia 8 has package names python-* 98 | # but requires python3 in shebang 99 | %if 0%{?mageia} >= 8 || 0%{?rhel} >= 8 || 0%{?suse_version} >= 1600 100 | %define python_path %{_bindir}/python3 101 | %else 102 | %define python_path %{_bindir}/%{use_python} 103 | %endif 104 | 105 | # avoid code duplication 106 | %define scm_common_dep \ 107 | Requires: obs-service-obs_scm-common = %version-%release \ 108 | %{nil} 109 | 110 | %define scm_dependencies \ 111 | Requires: git-core \ 112 | %if 0%{?suse_version} >= 1315 \ 113 | Recommends: obs-service-download_files \ 114 | Recommends: %{use_python}-keyring \ 115 | Recommends: %{use_python}-keyrings.alt \ 116 | Suggests: bzr \ 117 | Suggests: mercurial \ 118 | Suggests: subversion \ 119 | %endif \ 120 | %{nil} 121 | 122 | ######## END OF MACROS AND FUN ################################### 123 | 124 | %define pkg_name obs-service-tar_scm 125 | Name: %{pkg_name}%{nsuffix} 126 | %define version_unconverted 0.10.46 127 | Version: 0.10.46 128 | Release: 0 129 | Summary: An OBS source service: create tar ball from svn/git/hg 130 | License: GPL-2.0-or-later 131 | Group: Development/Tools/Building 132 | URL: https://github.com/openSUSE/obs-service-tar_scm 133 | Source: %{pkg_name}-%{version}.tar.gz 134 | 135 | # Fix build on Ubuntu by disabling mercurial tests, not applied in rpm 136 | # based distributions 137 | #Patch0: 0001-Debianization-disable-running-mercurial-tests.patch 138 | 139 | %if %{with obs_scm_testsuite} 140 | BuildRequires: %{locale_package} 141 | BuildRequires: %{pkg_name} = %{version} 142 | BuildRequires: %{use_python}-keyring 143 | BuildRequires: %{use_python}-keyrings.alt 144 | BuildRequires: %{use_python}-six 145 | BuildRequires: bzr 146 | BuildRequires: git-core 147 | BuildRequires: gpg 148 | BuildRequires: mercurial 149 | BuildRequires: subversion 150 | %if !%{with python3} 151 | BuildRequires: %{use_python}-mock 152 | %endif 153 | %endif 154 | 155 | BuildRequires: %{locale_package} 156 | BuildRequires: %{pyyaml_package} 157 | %if %{with needs_external_argparse} 158 | BuildRequires: %{use_python}-argparse 159 | %endif 160 | BuildRequires: %{use_python}-dateutil 161 | # Why do we need this? we dont use it as runtime requires later 162 | BuildRequires: %{use_python}-lxml 163 | 164 | %if %{with python3} 165 | BuildRequires: %{use_python}%{_pkg_base} 166 | # Fix missing Requires in python3-pbr in Leap42.3 167 | BuildRequires: %{use_python}-setuptools 168 | %if 0%{?suse_version} 169 | BuildRequires: python-rpm-macros 170 | %endif 171 | %else 172 | BuildRequires: python >= 2.6 173 | %endif 174 | %scm_common_dep 175 | %scm_dependencies 176 | # 177 | # 178 | # 179 | BuildRoot: %{_tmppath}/%{name}-%{version}-build 180 | BuildArch: noarch 181 | Requires: %{python_path} 182 | 183 | %description 184 | This is a source service for openSUSE Build Service. 185 | 186 | It supports downloading from svn, git, hg and bzr repositories. 187 | 188 | %package -n obs-service-obs_scm-common 189 | Summary: Common parts of SCM handling services 190 | Group: Development/Tools/Building 191 | Requires: %{locale_package} 192 | Requires: %{pyyaml_package} 193 | Requires: %{use_python}-dateutil 194 | %if %{with needs_external_argparse} 195 | Requires: %{use_python}-argparse 196 | %endif 197 | 198 | %description -n obs-service-obs_scm-common 199 | This is a source service for openSUSE Build Service. 200 | 201 | It supports downloading from svn, git, hg and bzr repositories. 202 | 203 | This package holds the shared files for different services. 204 | 205 | %package -n obs-service-tar 206 | Summary: Creates a tar archive from local directory 207 | Group: Development/Tools/Building 208 | Provides: obs-service-tar_scm:/usr/lib/obs/service/tar.service 209 | %scm_common_dep 210 | 211 | %description -n obs-service-tar 212 | Creates a tar archive from local directory 213 | 214 | %package -n obs-service-obs_scm 215 | Summary: Creates a OBS cpio from a remote SCM resource 216 | Group: Development/Tools/Building 217 | Provides: obs-service-tar_scm:/usr/lib/obs/service/obs_scm.service 218 | %scm_common_dep 219 | %scm_dependencies 220 | 221 | %description -n obs-service-obs_scm 222 | Creates a OBS cpio from a remote SCM resource. 223 | 224 | This can be used to work directly in local git checkout and can be packaged 225 | into a tar ball during build time. 226 | 227 | %package -n obs-service-appimage 228 | Summary: Handles source downloads defined in appimage.yml files 229 | Group: Development/Tools/Building 230 | %scm_common_dep 231 | %scm_dependencies 232 | 233 | %description -n obs-service-appimage 234 | Experimental appimage support: This parses appimage.yml files for SCM 235 | resources and packages them. 236 | 237 | %package -n obs-service-snapcraft 238 | Summary: Handles source downloads defined in snapcraft.yaml files 239 | Group: Development/Tools/Building 240 | Provides: obs-service-tar_scm:/usr/lib/obs/service/snapcraft.service 241 | %scm_common_dep 242 | %scm_dependencies 243 | 244 | %description -n obs-service-snapcraft 245 | Experimental snapcraft support: This parses snapcraft.yaml files for SCM 246 | resources and packages them. 247 | 248 | %if 0%{?enable_gbp} 249 | %package -n obs-service-gbp 250 | Summary: Creates Debian source artefacts from a Git repository 251 | Group: Development/Tools/Building 252 | Requires: git-buildpackage >= 0.6.0 253 | Requires: obs-service-obs_scm-common = %version-%release 254 | %if 0%{?enable_gbp} 255 | Provides: obs-service-tar_scm:/usr/lib/obs/service/obs_gbp.service 256 | %endif 257 | 258 | %description -n obs-service-gbp 259 | Debian git-buildpackage workflow support: uses gbp to create Debian 260 | source artefacts (.dsc, .origin.tar.gz and .debian.tar.gz if non-native). 261 | %endif 262 | 263 | %prep 264 | %setup -q -n obs-service-tar_scm-%version 265 | 266 | %build 267 | 268 | %install 269 | %if %{without obs_scm_testsuite} 270 | make install DESTDIR="%{buildroot}" PREFIX="%{_prefix}" SYSCFG="%{_sysconfdir}" PYTHON="%{python_path}" WITH_GBP="%{enable_gbp}" 271 | %if %{with python3} 272 | # Doing %%python3_fix_shebang_path old fashioned way for the backward compatibility 273 | sed -i "1s@#\\!.*python\S*@#\\!$(realpath %__python3)@" \ 274 | %{buildroot}%{_prefix}/lib/obs/service/tar_scm 275 | %endif 276 | %else 277 | 278 | # moved conditional to the top as it helps to have it all in one place and only rely on the bcond_with here. 279 | %check 280 | # No need to run PEP8 tests here; that would require a potentially 281 | # brittle BuildRequires: python-pep8, and any style issues are already 282 | # caught by Travis CI. 283 | make %{use_test} 284 | %endif 285 | 286 | %if %{without obs_scm_testsuite} 287 | %files 288 | %defattr(-,root,root) 289 | %{_prefix}/lib/obs/service/tar_scm.service 290 | 291 | %files -n obs-service-obs_scm-common 292 | %defattr(-,root,root) 293 | %license COPYING 294 | %dir %{_prefix}/lib/obs 295 | %dir %{_prefix}/lib/obs/service 296 | %{_prefix}/lib/obs/service/TarSCM 297 | %{_prefix}/lib/obs/service/tar_scm 298 | %dir %{_sysconfdir}/obs 299 | %dir %{_sysconfdir}/obs/services 300 | %verify (not user group) %dir %{_sysconfdir}/obs/services/tar_scm.d 301 | %config(noreplace) %{_sysconfdir}/obs/services/* 302 | %ghost %dir %{_sysconfdir}/obs/services/tar_scm.d/python_keyring 303 | 304 | %files -n obs-service-tar 305 | %defattr(-,root,root) 306 | %{_prefix}/lib/obs/service/tar 307 | %{_prefix}/lib/obs/service/tar.service 308 | 309 | %files -n obs-service-obs_scm 310 | %defattr(-,root,root) 311 | %{_prefix}/lib/obs/service/obs_scm 312 | %{_prefix}/lib/obs/service/obs_scm.service 313 | 314 | %files -n obs-service-appimage 315 | %defattr(-,root,root) 316 | %{_prefix}/lib/obs/service/appimage* 317 | 318 | %files -n obs-service-snapcraft 319 | %defattr(-,root,root) 320 | %{_prefix}/lib/obs/service/snapcraft* 321 | 322 | %if 0%{?enable_gbp} 323 | %files -n obs-service-gbp 324 | %defattr(-,root,root) 325 | %{_prefix}/lib/obs/service/obs_gbp* 326 | %endif 327 | %endif 328 | 329 | %changelog 330 | -------------------------------------------------------------------------------- /obs_gbp: -------------------------------------------------------------------------------- 1 | tar_scm.py -------------------------------------------------------------------------------- /obs_scm: -------------------------------------------------------------------------------- 1 | tar_scm.py -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | lxml 2 | mock; python_version < '3.3' 3 | pep8 4 | python-dateutil 5 | PyYAML 6 | six 7 | pylint < 2.14 8 | flake8 9 | -------------------------------------------------------------------------------- /snapcraft: -------------------------------------------------------------------------------- 1 | tar_scm.py -------------------------------------------------------------------------------- /snapcraft.service: -------------------------------------------------------------------------------- 1 | 2 | handle sources specified in snapcraft.yaml 3 | This service needs to be executed to download sources according to snapcraft.yaml file. It also patches the snapcraft tile to use local sources during build. 4 | 5 | Specify whether to include git submodules. Default is 'enable'. main or master is override the specified commit with master or main branch. 6 | enable 7 | master 8 | main 9 | disable 10 | 11 | 12 | Specify whether to include git lfs blobs. Default is 'disable'. 13 | enable 14 | disable 15 | 16 | 17 | Specify Whether or not to check server certificate against installed CAs. Default is 'enable'. 18 | enable 19 | disable 20 | 21 | 22 | -------------------------------------------------------------------------------- /tar: -------------------------------------------------------------------------------- 1 | tar_scm.py -------------------------------------------------------------------------------- /tar.service: -------------------------------------------------------------------------------- 1 | 2 | Create a tarball from SCM repository 3 | This service is packaging a directory structure into a tar ball. The directory gets created via the obs_scm service usually. This service reads the .obsinfo file to learn how to create the tar ball. 4 | 5 | Specify .obsinfo file to create a tar ball. 6 | 7 | 8 | Specify version to be used in tarball. Defaults to automatically detected value formatted by versionformat parameter. Disabling the additional version string can be done by setting it to "_none_" string. 9 | 10 | 11 | Specify a base version as prefix. 12 | 13 | 14 | Specify name of package, which is used together with version to determine tarball name. 15 | 16 | 17 | Specify suffix name of package, which is used together with filename to determine tarball name. 18 | 19 | 20 | Package the metadata of SCM to allow the user or OBS to update after un-tar. 21 | yes 22 | 23 | 24 | -------------------------------------------------------------------------------- /tar_scm.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # A simple script to checkout or update a svn or git repo as source service 4 | # 5 | # (C) 2010 by Adrian Schroeter 6 | # (C) 2014 by Jan Blunck (Python rewrite) 7 | # (C) 2016 by Adrian Schroeter (OBS cpio support) 8 | # 9 | # This program is free software; you can redistribute it and/or 10 | # modify it under the terms of the GNU General Public License 11 | # as published by the Free Software Foundation; either version 2 12 | # of the License, or (at your option) any later version. 13 | # See http://www.gnu.org/licenses/gpl-2.0.html for full license text. 14 | 15 | import os 16 | import sys 17 | 18 | sys.path.append(os.path.dirname(os.path.abspath(__file__))) # noqa 19 | 20 | # These two modules have to be imported after sys.path.append because 21 | # the additional path is where the modules are found 22 | import TarSCM # pylint: disable=wrong-import-position; # noqa: E402 23 | 24 | 25 | def main(): 26 | TarSCM.run() 27 | 28 | 29 | if __name__ == '__main__': 30 | main() 31 | -------------------------------------------------------------------------------- /tar_scm.rc: -------------------------------------------------------------------------------- 1 | # 2 | # Define a cache directory here to avoid repeating downloads of the 3 | # same file. This works on the server and on the client side. 4 | # 5 | # Note that this file can be overridden with per-user config placed 6 | # in ~/.obs/tar_scm 7 | # 8 | # WARNING: you need to create three directories inside, when changing from default: 9 | # mkdir -p repo{,url} incoming 10 | # 11 | #CACHEDIRECTORY="/var/cache/obs/tar_scm" 12 | -------------------------------------------------------------------------------- /tar_scm.service.in: -------------------------------------------------------------------------------- 1 | ===OBS_ONLY 2 | 3 | Create a special OBS archive from a SCM 4 | This service uses a SCM client to checkout or update from a given repository. Supported are svn, git, hg and bzr. The result will archived in a format which can be stored 5 | incremental by the OBS source server, currently a cpio format. This archive will be extracted again inside of the build root. 6 | === 7 | 8 | ===TAR_ONLY 9 | 10 | Create a tarball from SCM repository 11 | This service uses a SCM client to checkout or update from a given repository. Supported are svn, git, hg and bzr. 12 | === 13 | 14 | ===GBP_ONLY 15 | 16 | Create Debian source artefacts from SCM repository 17 | This service uses a Git to checkout or update from a given repository and create the Debian source artefacts (.dsc, .origin.tar.gz and .debian.tar.gz if non-native). Can only be used with (and implies) --scm git. 18 | 19 | Parameters passed to git-buildpackage. Default is '-nc -uc -us -S'. 20 | 21 | 22 | Append OBS release number. Default is 'disable'. 23 | enable 24 | disable 25 | 26 | === 27 | 28 | 29 | Specify SCM to use. 30 | svn 31 | git 32 | hg 33 | bzr 34 | 35 | 36 | 37 | Specify URL to checkout. Allowed URL types: (http/https/ftp/ftps/git/ssh) 38 | 39 | 40 | 41 | 42 | Specify the username to be used for authentication to the repository. 43 | 44 | 45 | 46 | 47 | Specify the passphrase to decrypt credentials from the python keyring. 48 | To store credentials please use the following command lines: 49 | "sudo chown obsservicerun:obsrun /etc/obs/services/tar_scm.d" 50 | "sudo -H -u obsservicerun XDG_DATA_HOME=/etc/obs/services/tar_scm.d keyring -b keyrings.alt.file.EncryptedKeyring set URL username" 51 | Its only available for the following combination of SCM / protocols: 52 | - git: ftp(s),http(s) 53 | - svn 54 | - bzr: bzr,ftp,http(s) 55 | - hg: http(s) 56 | 57 | 58 | 59 | Package just a subdirectory. 60 | 61 | 62 | 63 | Specify version to be used in tarball. Defaults to automatically detected value formatted by versionformat parameter (_auto_). 64 | Use _none_ to disable version rewriting and use what is defined in the spec or debian/changelog. 65 | 66 | 67 | 68 | 69 | Auto-generate version from checked out source using this format 70 | string. This parameter is used if the 'version' parameter is not 71 | specified. 72 | 73 | For git, the value is passed to git log --date=short --pretty=format:... 74 | for the topmost commit, and the output from git is cleaned up to 75 | remove some unhelpful characters. Here are some useful examples of 76 | strings which are expanded, see the git-log documentation for more. 77 | 78 | %ct Commit time as a UNIX timestamp, e.g. 1384855776. 79 | This is the default. 80 | 81 | %at Author time as a UNIX timestamp, e.g. 1384855776. 82 | 83 | %cd Commit date in YYYYMMDD format, e.g. 20131119 84 | 85 | %ad Author date in YYYYMMDD format, e.g. 20131119 86 | 87 | %h Abbreviated hash, e.g. cc62c54 88 | 89 | @PARENT_TAG@ The first tag that is reachable, e.g. v0.2.3 90 | 91 | @TAG_OFFSET@ The commit count since @PARENT_TAG@, e.g. 9 92 | 93 | For hg, the value is passed to hg log --template=.... See the 94 | hg documentation for more information. The default is '{rev}' 95 | 96 | For bzr and svn, '%r' is expanded to the revision, and is the default. 97 | 98 | 99 | 100 | 101 | Regex used to rewrite the version which is applied post versionformat. For 102 | example, to remove a tag prefix of "v" the regex "v(.*)" could be used. 103 | See the versionrewrite-replacement parameter. 104 | 105 | 106 | 107 | 108 | Replacement applied to rewrite pattern. Typically backreferences are 109 | useful and as such defaults to \1. 110 | 111 | 112 | 113 | Specify a base version as prefix. 114 | 115 | 116 | 117 | With this parameter you can specify a glob pattern (e.g. v*) to filter 118 | relevant tags in your project e.g. if you use @PARENT_TAG@. 119 | 120 | 121 | 122 | 123 | This parameter allows overriding the tag that is being used for 124 | computing @TAG_OFFSET@. 125 | 126 | 127 | 128 | 129 | Specify revision of source to check out. 130 | 131 | When using git, revision may refer to any of the following: 132 | 133 | * explicit SHA1: a1b2c3d4.... 134 | - the SHA1 must be reachable from a default clone/fetch 135 | (generally, must be reachable from some branch or tag on the 136 | remote). 137 | - set by: git checkout ${SHA1} 138 | 139 | * short branch name: "master", "devel" etc. 140 | - set by: git checkout ${branch} 141 | git pull 142 | 143 | * explicit ref: refs/heads/master, refs/tags/v1.2.3, 144 | refs/changes/49/11249/1 145 | - set by: git fetch ${url} ${revision}:${revision} 146 | git checkout ${revision} 147 | 148 | * the first tag that is reachable via git describe on the default branch: @PARENT_TAG@ 149 | 150 | 151 | 152 | Specify name of package, which is used together with version to determine tarball name. 153 | 154 | 155 | Specify suffix name of package, which is used together with filename to determine tarball name. 156 | 157 | 158 | Specify glob pattern to exclude when creating the tarball. 159 | 160 | 161 | Specify subset of files/subdirectories to pack in the tarball. 162 | 163 | 164 | Specify a file/glob to be exported directly. Useful for build descriptions like spec files 165 | which get maintained in the SCM. Can be used multiple times. 166 | 167 | 168 | Package the metadata of SCM to allow the user or OBS to update after un-tar. Please be aware that this parameter has precedence over the "exclude" paramter. 169 | yes 170 | 171 | 172 | Obsolete parameter which will be ignored. 173 | 174 | 175 | Specify whether to include git submodules. Default is 'enable'. main or master is override the specified commit with master or main branch. 176 | enable 177 | master 178 | main 179 | disable 180 | 181 | ===OBS_ONLY 182 | 183 | Specify whether to include git-lfs blobs. Default is 'disable'. 184 | enable 185 | disable 186 | 187 | === 188 | 189 | Specify Whether or not to check server certificate against installed CAs. Default is 'enable'. 190 | enable 191 | disable 192 | 193 | 194 | Specify whether to generate changes file entries from SCM commit log since a given parent revision (see changesrevision). Default is 'disable'. 195 | enable 196 | disable 197 | 198 | 199 | Specify author of the changes file entry to be written. Defaults to first email entry in ~/.oscrc, or "obs-service-tar-scm@invalid" if there is no .oscrc found. 200 | 201 | 202 | DEPRECATED - Please use "encoding" instead. Set locale while execution of service 203 | 204 | 205 | Set encoding while execution of service 206 | 207 | 208 | Use the latest signed commit on a branch 209 | 210 | 211 | Use the latest signed tag on a branch 212 | 213 | 214 | File which contains maintainers pubkeys (only used with '--latest-signed-*') 215 | 216 | 217 | Do not add version to output file. 218 | 219 | 220 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openSUSE/obs-service-tar_scm/bfbf96f2b1dcd8e1c0743ec69e5c142807252981/tests/__init__.py -------------------------------------------------------------------------------- /tests/archiveobscpiotestcases.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from __future__ import print_function 3 | 4 | import os 5 | import re 6 | import inspect 7 | import shutil 8 | import unittest 9 | import io 10 | import six 11 | import yaml 12 | 13 | import TarSCM 14 | 15 | from TarSCM.scm.git import Git 16 | from TarSCM.archive import ObsCpio 17 | 18 | from tests.gitfixtures import GitFixtures 19 | from tests.scmlogs import ScmInvocationLogs 20 | 21 | 22 | class ArchiveOBSCpioTestCases(unittest.TestCase): 23 | def setUp(self): 24 | self.cli = TarSCM.Cli() 25 | self.tasks = TarSCM.Tasks(self.cli) 26 | self.tests_dir = os.path.abspath(os.path.dirname(__file__)) 27 | self.tmp_dir = os.path.join(self.tests_dir, 'tmp') 28 | self.fixtures_dir = os.path.join(self.tests_dir, 'fixtures', 29 | self.__class__.__name__) 30 | 31 | self.cli.parse_args(['--outdir', '.']) 32 | os.environ['CACHEDIRECTORY'] = '' 33 | 34 | def test_obscpio_create_archive(self): 35 | tc_name = inspect.stack()[0][3] 36 | cl_name = self.__class__.__name__ 37 | c_dir = os.path.join(self.tmp_dir, tc_name) 38 | f_dir = os.path.join(self.fixtures_dir, tc_name, 'repo') 39 | shutil.copytree(f_dir, c_dir) 40 | scmlogs = ScmInvocationLogs('git', c_dir) 41 | scmlogs.nextlog('start-test') 42 | fixture = GitFixtures(c_dir, scmlogs) 43 | fixture.init() 44 | scm_object = Git(self.cli, self.tasks) 45 | scm_object.clone_dir = fixture.repo_path 46 | scm_object.arch_dir = fixture.repo_path 47 | outdir = os.path.join(self.tmp_dir, cl_name, tc_name, 48 | 'out') 49 | 50 | self.cli.outdir = outdir 51 | arch = ObsCpio() 52 | os.makedirs(outdir) 53 | version = '0.1.1' 54 | (dst, chgv, bname) = self.tasks._dstname(scm_object, version) 55 | arch.create_archive( 56 | scm_object, 57 | cli = self.cli, 58 | basename = bname, 59 | dstname = dst, 60 | version = chgv 61 | ) 62 | cpiofile = os.path.join(outdir, "%s-%s.obscpio" % (bname, version)) 63 | infofile = os.path.join(outdir, bname + ".obsinfo") 64 | self.assertTrue(os.path.isfile(cpiofile)) 65 | self.assertTrue(os.path.isfile(infofile)) 66 | with io.open(infofile, 'r', encoding='UTF-8') as fhl: 67 | data = yaml.safe_load(fhl) 68 | self.assertDictEqual( 69 | data, { 70 | 'name': bname, 71 | 'version': chgv, 72 | 'mtime': 1234567890, 73 | 'commit': data['commit']}) 74 | 75 | def test_obscpio_extract_of(self): 76 | ''' 77 | Test obscpio to extract one file from archive 78 | ''' 79 | tc_name = inspect.stack()[0][3] 80 | cl_name = self.__class__.__name__ 81 | 82 | repodir = os.path.join(self.fixtures_dir, tc_name, 'repo') 83 | files = ["test.spec"] 84 | outdir = os.path.join(self.tmp_dir, cl_name, tc_name, 'out') 85 | arch = ObsCpio() 86 | os.makedirs(outdir) 87 | arch.extract_from_archive(repodir, files, outdir) 88 | for fname in files: 89 | self.assertTrue(os.path.exists( 90 | os.path.join(outdir, fname))) 91 | 92 | def test_obscpio_extract_mf(self): 93 | ''' 94 | Test obscpio to extract multiple files from archive 95 | ''' 96 | tc_name = inspect.stack()[0][3] 97 | cl_name = self.__class__.__name__ 98 | 99 | repodir = os.path.join(self.fixtures_dir, tc_name, 'repo') 100 | files = ["test.spec", 'Readme.md'] 101 | outdir = os.path.join(self.tmp_dir, cl_name, tc_name, 'out') 102 | arch = ObsCpio() 103 | os.makedirs(outdir) 104 | arch.extract_from_archive(repodir, files, outdir) 105 | for fname in files: 106 | self.assertTrue(os.path.exists( 107 | os.path.join(outdir, fname))) 108 | 109 | def test_obscpio_extract_nef(self): 110 | ''' 111 | Test obscpio to extract non existant file from archive 112 | ''' 113 | tc_name = inspect.stack()[0][3] 114 | cl_name = self.__class__.__name__ 115 | 116 | repodir = os.path.join(self.fixtures_dir, tc_name, 'repo') 117 | files = ['nonexistantfile'] 118 | outdir = os.path.join(self.tmp_dir, cl_name, tc_name, 'out') 119 | arch = ObsCpio() 120 | os.makedirs(outdir) 121 | six.assertRaisesRegex( 122 | self, 123 | SystemExit, 124 | re.compile('No such file or directory'), 125 | arch.extract_from_archive, 126 | repodir, 127 | files, 128 | outdir 129 | ) 130 | 131 | def test_obscpio_extract_d(self): 132 | ''' 133 | Test obscpio to extract directory from archive 134 | ''' 135 | tc_name = inspect.stack()[0][3] 136 | cl_name = self.__class__.__name__ 137 | 138 | repodir = os.path.join(self.fixtures_dir, tc_name, 'repo') 139 | files = ['dir1'] 140 | outdir = os.path.join(self.tmp_dir, cl_name, tc_name, 'out') 141 | arch = TarSCM.archive.ObsCpio() 142 | os.makedirs(outdir) 143 | six.assertRaisesRegex( 144 | self, 145 | IOError, 146 | re.compile('Is a directory:'), 147 | arch.extract_from_archive, 148 | repodir, 149 | files, 150 | outdir 151 | ) 152 | 153 | def test_obscpio_broken_link(self): 154 | tc_name = inspect.stack()[0][3] 155 | cl_name = self.__class__.__name__ 156 | c_dir = os.path.join(self.tmp_dir, tc_name) 157 | scmlogs = ScmInvocationLogs('git', c_dir) 158 | scmlogs.nextlog('start-test') 159 | fixture = GitFixtures(c_dir, scmlogs) 160 | fixture.init() 161 | scm_object = Git(self.cli, self.tasks) 162 | scm_object.clone_dir = fixture.repo_path 163 | scm_object.arch_dir = fixture.repo_path 164 | outdir = os.path.join(self.tmp_dir, cl_name, tc_name, 165 | 'out') 166 | cwd = os.getcwd() 167 | print("cwd = %s" % cwd) 168 | os.chdir(fixture.repo_path) 169 | os.symlink('non-existant-file', 'broken-link') 170 | fixture.run('add broken-link') 171 | fixture.run("commit -m 'added broken-link'") 172 | os.chdir(cwd) 173 | 174 | self.cli.outdir = outdir 175 | arch = ObsCpio() 176 | os.makedirs(outdir) 177 | arch.create_archive( 178 | scm_object, 179 | cli = self.cli, 180 | basename = 'test', 181 | dstname = 'test', 182 | version = '0.1.1' 183 | ) 184 | 185 | def test_extract_link_outside_repo(self): 186 | ''' 187 | Test if a link outside the repo gets detected 188 | ''' 189 | tc_name = inspect.stack()[0][3] 190 | cl_name = self.__class__.__name__ 191 | files = ['dir1/etc/passwd'] 192 | 193 | # create repodir 194 | repodir = os.path.join(self.tmp_dir, tc_name, 'repo') 195 | os.makedirs(repodir) 196 | 197 | # create outdir 198 | outdir = os.path.join(self.tmp_dir, cl_name, tc_name, 'out') 199 | os.makedirs(outdir) 200 | 201 | os.symlink("/", os.path.join(repodir, 'dir1')) 202 | arch = TarSCM.archive.ObsCpio() 203 | six.assertRaisesRegex( 204 | self, 205 | SystemExit, 206 | re.compile('tries to escape the repository'), 207 | arch.extract_from_archive, 208 | repodir, 209 | files, 210 | outdir 211 | ) 212 | 213 | def test_obscpio_extract_glob(self): 214 | ''' 215 | Test obscpio to extract file glob from archive 216 | ''' 217 | tc_name = inspect.stack()[0][3] 218 | cl_name = self.__class__.__name__ 219 | 220 | repodir = os.path.join(self.fixtures_dir, tc_name, 'repo') 221 | files = ["test.*"] 222 | files_expected = ["test.spec", "test.rpmlintrc"] 223 | outdir = os.path.join(self.tmp_dir, cl_name, tc_name, 'out') 224 | arch = ObsCpio() 225 | os.makedirs(outdir) 226 | arch.extract_from_archive(repodir, files, outdir) 227 | for fname in files_expected: 228 | self.assertTrue(os.path.exists( 229 | os.path.join(outdir, fname))) 230 | -------------------------------------------------------------------------------- /tests/bzrfixtures.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import os 4 | 5 | from fixtures import Fixtures 6 | from utils import run_bzr 7 | 8 | 9 | class BzrFixtures(Fixtures): 10 | """Methods to create and populate a bzr repository. 11 | 12 | bzr tests use this class in order to have something to test against. 13 | """ 14 | 15 | def init(self): 16 | self.create_repo() 17 | self.create_commits(2) 18 | 19 | def run(self, cmd): 20 | return run_bzr(self.repo_path, cmd) 21 | 22 | def create_repo(self): 23 | os.makedirs(self.repo_path) 24 | os.chdir(self.repo_path) 25 | self.safe_run('init') 26 | self.safe_run('whoami "%s"' % self.name_and_email) 27 | self.wdir = self.repo_path 28 | print("created repo %s" % self.repo_path) 29 | 30 | def record_rev(self, *args): 31 | rev_num = args[0] 32 | self.revs[rev_num] = str(rev_num) 33 | self.scmlogs.annotate("Recorded rev %d" % rev_num) 34 | 35 | def get_committer_date(self): 36 | '''There seems to be no way to create a commit with a given timestamp 37 | set for Bazar.''' 38 | return '' 39 | -------------------------------------------------------------------------------- /tests/bzrtests.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from commontests import CommonTests 4 | from bzrfixtures import BzrFixtures 5 | 6 | 7 | class BzrTests(CommonTests): 8 | 9 | """Unit tests for 'tar_scm --scm bzr'. 10 | 11 | bzr-specific tests are in this class. Other shared tests are 12 | included via the class inheritance hierarchy. 13 | """ 14 | 15 | scm = 'bzr' 16 | initial_clone_command = 'bzr checkout' 17 | update_cache_command = 'bzr update' 18 | sslverify_false_args = '-Ossl.cert_reqs=None' 19 | fixtures_class = BzrFixtures 20 | 21 | def default_version(self): 22 | return self.rev(2) 23 | 24 | def test_versionformat_rev(self): 25 | self.tar_scm_std('--versionformat', 'myrev%r.svn') 26 | self.assertTarOnly(self.basename(version='myrev2.svn')) 27 | 28 | def test_version_versionformat(self): 29 | self.tar_scm_std('--version', '3.0', '--versionformat', 'myrev%r.svn') 30 | self.assertTarOnly(self.basename(version='myrev2.svn')) 31 | 32 | def test_versionformat_revision(self): 33 | self.fixtures.create_commits(4) 34 | self.tar_scm_std('--versionformat', 'foo%r', '--revision', self.rev(2)) 35 | basename = self.basename(version='foo2') 36 | tar = self.assertTarOnly(basename) 37 | self.assertTarMemberContains(tar, basename + '/a', '2') 38 | 39 | def assertDirentsMtime(self, entries): 40 | '''Skip this test with bazaar because there seem to be no way to create 41 | commits with a given timestamp.''' 42 | return True 43 | -------------------------------------------------------------------------------- /tests/commontests.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from pprint import pformat 4 | import os 5 | import tarfile 6 | import six 7 | 8 | from tests.testassertions import TestAssertions 9 | from tests.testenv import TestEnvironment 10 | from tests.utils import mkfreshdir, run_cmd 11 | 12 | 13 | class CommonTests(TestEnvironment, TestAssertions): 14 | 15 | """Unit tests common to all version control systems. 16 | 17 | Unit tests here are not specific to any particular version control 18 | system, and will be run for all of git / hg / svn / bzr. 19 | """ 20 | scm = None 21 | 22 | def basename(self, name='repo', version=None): 23 | if version is None: 24 | version = self.default_version() # pylint: disable=E1101 25 | return '%s-%s' % (name, version) 26 | 27 | def test_plain(self): 28 | self.tar_scm_std() 29 | self.assertTarOnly(self.basename()) 30 | 31 | def test_symlink(self): 32 | self.fixtures.create_commits(1) 33 | self.tar_scm_std('--versionformat', '3', 34 | '--revision', self.rev(3)) 35 | basename = self.basename(version=3) 36 | tar_handle = self.assertTarOnly(basename) 37 | # tarfile.extractfile() in python 2.6 is broken when extracting 38 | # relative symlinks as a file object so we construct linkname manually 39 | member = tar_handle.getmember(basename + '/c') 40 | self.assertTrue(member.issym()) 41 | self.assertEqual(member.linkname, 'a') 42 | linkname = '/'.join([os.path.dirname(member.name), member.linkname]) 43 | self.assertTarMemberContains(tar_handle, linkname, '3') 44 | 45 | def test_broken_symlink(self): 46 | self.fixtures.create_commit_broken_symlink() 47 | self.tar_scm_std('--versionformat', '3', 48 | '--revision', self.rev(3)) 49 | basename = self.basename(version=3) 50 | tar_handle = self.assertTarOnly(basename) 51 | member = tar_handle.getmember(basename + '/c') 52 | self.assertTrue(member.issym()) 53 | six.assertRegex(self, member.linkname, '[/.]*/nir/va/na$') 54 | 55 | def test_tar_exclude(self): 56 | self.tar_scm_std('--exclude', 'a', '--exclude', 'c') 57 | tar_file = os.path.join(self.outdir, self.basename()+'.tar') 58 | tar = tarfile.open(tar_file) 59 | tarents = tar.getnames() 60 | expected = [self.basename(), 61 | self.basename() + '/subdir', 62 | self.basename() + '/subdir/b'] 63 | self.assertTrue(tarents == expected) 64 | 65 | def test_tar_exclude_re(self): 66 | self.tar_scm_std('--exclude', '(a|c)') 67 | tar_file = os.path.join(self.outdir, self.basename()+'.tar') 68 | tar = tarfile.open(tar_file) 69 | tarents = tar.getnames() 70 | expected = [self.basename(), 71 | self.basename() + '/subdir', 72 | self.basename() + '/subdir/b'] 73 | self.assertTrue(tarents == expected) 74 | 75 | def test_tar_include(self): 76 | self.tar_scm_std('--include', self.fixtures.subdir) 77 | tar_file = os.path.join(self.outdir, self.basename()+'.tar') 78 | tar = tarfile.open(tar_file) 79 | tarents = tar.getnames() 80 | expected = [self.basename(), 81 | self.basename() + '/subdir', 82 | self.basename() + '/subdir/b'] 83 | self.assertTrue(tarents == expected) 84 | 85 | def test_obs_scm_exclude(self): 86 | self.tar_scm_std('--exclude', 'a', '--exclude', 'c', '--use-obs-scm', 'True') 87 | cpio = os.path.join(self.outdir, self.basename()+'.obscpio') 88 | cmd = "cpio -it < "+cpio 89 | (stdout, _stderr, _ret) = run_cmd(cmd) 90 | got = stdout.decode().split("\n") 91 | got.pop() 92 | expected = [self.basename() + '/subdir', 93 | self.basename() + '/subdir/b'] 94 | self.assertTrue(got == expected) 95 | 96 | def test_obs_scm_include(self): 97 | self.tar_scm_std('--include', self.fixtures.subdir, '--use-obs-scm', 'True') 98 | cpio = os.path.join(self.outdir, self.basename()+'.obscpio') 99 | cmd = "cpio -it < "+cpio 100 | (stdout, _stderr, _ret) = run_cmd(cmd) 101 | got = stdout.decode().split("\n") 102 | got.pop() 103 | expected = [self.basename() + '/subdir', 104 | self.basename() + '/subdir/b'] 105 | self.assertTrue(got == expected) 106 | 107 | 108 | def test_absolute_subdir(self): 109 | (_stdout, stderr, _ret) = self.tar_scm_std_fail('--subdir', '/') 110 | six.assertRegex( 111 | self, stderr, "Absolute path '/' is not allowed for --subdir") 112 | 113 | def test_subdir_parent(self): 114 | for path in ('..', '../', '../foo', 'foo/../../bar'): 115 | (_stdout, stderr, _ret) = self.tar_scm_std_fail('--subdir', path) 116 | six.assertRegex( 117 | self, stderr, "--subdir path '%s' must stay within repo" % path) 118 | 119 | def test_extract_parent(self): 120 | for path in ('..', '../', '../foo', 'foo/../../bar'): 121 | (_stdout, stderr, _ret) = self.tar_scm_std_fail('--extract', path) 122 | six.assertRegex( 123 | self, stderr, '--extract is not allowed to contain ".."') 124 | 125 | def test_filename(self): 126 | for path in ('/tmp/somepkg.tar', '../somepkg.tar'): 127 | (_stdout, stderr, _ret) = self.tar_scm_std_fail('--filename', path) 128 | six.assertRegex( 129 | self, stderr, '--filename must not specify a path') 130 | 131 | def test_subdir(self): 132 | self.tar_scm_std('--subdir', self.fixtures.subdir) 133 | self.assertTarOnly(self.basename(), tarchecker=self.assertSubdirTar) 134 | 135 | def test_history_depth_obsolete(self): 136 | (stdout, _stderr, _ret) = self.tar_scm_std('--history-depth', '1') 137 | six.assertRegex(self, stdout, 'obsolete') 138 | 139 | def test_myfilename(self): 140 | name = 'myfilename' 141 | self.tar_scm_std('--filename', name) 142 | self.assertTarOnly(self.basename(name=name)) 143 | 144 | def test_version(self): 145 | version = '0.5' 146 | self.tar_scm_std('--version', version) 147 | self.assertTarOnly(self.basename(version=version)) 148 | 149 | def test_filename_version(self): 150 | filename = 'myfilename' 151 | version = '0.6' 152 | self.tar_scm_std('--filename', filename, '--version', version) 153 | self.assertTarOnly(self.basename(filename, version)) 154 | 155 | def test_filename_without_version(self): 156 | filename = 'myfilename' 157 | self.fixtures.create_commits(1) 158 | self.tar_scm_std('--filename', filename, '--version', '_none_') 159 | self.assertTarOnly(filename) 160 | 161 | def test_revision_nop(self): 162 | self.tar_scm_std('--revision', self.rev(2)) 163 | tar_handle = self.assertTarOnly(self.basename()) 164 | self.assertTarMemberContains(tar_handle, self.basename() + '/a', '2') 165 | 166 | def test_revision(self): 167 | self._revision() 168 | 169 | def test_revision_lang_de(self): 170 | os.putenv('LANG', 'de_DE.UTF-8') 171 | os.environ['LANG'] = 'de_DE.UTF-8' 172 | self._revision() 173 | os.unsetenv('LANG') 174 | os.environ['LANG'] = '' 175 | 176 | def test_revision_no_cache(self): 177 | self._revision(use_cache=False) 178 | 179 | def test_revision_subdir(self): 180 | self._revision(use_subdir=True) 181 | 182 | def test_revision_subdir_no_cache(self): 183 | self._revision(use_cache=False, use_subdir=True) 184 | 185 | def _revision(self, use_cache=True, use_subdir=False): 186 | """ 187 | Check that the right revision is packaged up, regardless of 188 | whether new commits have been introduced since previous runs. 189 | """ 190 | version = '3.0' 191 | args_tag2 = [ 192 | '--version', version, 193 | '--revision', self.rev(2), 194 | ] 195 | if use_subdir: 196 | args_tag2 += ['--subdir', self.fixtures.subdir] 197 | self._sequential_calls_with_revision( 198 | version, 199 | [ 200 | (0, args_tag2, '2', False), 201 | (0, args_tag2, '2', use_cache), 202 | (2, args_tag2, '2', use_cache), 203 | (0, args_tag2, '2', use_cache), 204 | (2, args_tag2, '2', use_cache), 205 | (0, args_tag2, '2', use_cache), 206 | ], 207 | use_cache 208 | ) 209 | 210 | def test_rev_alter(self): 211 | self._revision_master_alternating() 212 | 213 | def test_rev_alter_no_cache(self): 214 | self._revision_master_alternating(use_cache=False) 215 | 216 | def test_rev_alter_subdir(self): 217 | self._revision_master_alternating(use_subdir=True) 218 | 219 | def test_rev_alter_subdir_no_cache(self): 220 | self._revision_master_alternating(use_cache=False, use_subdir=True) 221 | 222 | def _revision_master_alternating(self, use_cache=True, use_subdir=False): 223 | """ 224 | Call tar_scm 7 times, alternating between a specific revision 225 | and the default branch (master), and checking the results each 226 | time. New commits are created before some of the invocations. 227 | """ 228 | version = '4.0' 229 | args_head = [ 230 | '--version', version, 231 | ] 232 | if use_subdir: 233 | args_head += ['--subdir', self.fixtures.subdir] 234 | 235 | args_tag2 = args_head + ['--revision', self.rev(2)] 236 | self._sequential_calls_with_revision( 237 | version, 238 | [ 239 | (0, args_tag2, '2', False), 240 | (0, args_head, '2', use_cache), 241 | (2, args_tag2, '2', use_cache), 242 | (0, args_head, '4', use_cache), 243 | (2, args_tag2, '2', use_cache), 244 | (0, args_head, '6', use_cache), 245 | (0, args_tag2, '2', use_cache), 246 | ], 247 | use_cache 248 | ) 249 | 250 | def _sequential_calls_with_revision(self, version, calls, use_cache=True): 251 | """ 252 | Call tar_scm a number of times, optionally creating some 253 | commits before each invocation, and checking that the result 254 | contains the right revision after each invocation. 255 | """ 256 | mkfreshdir(self.pkgdir) 257 | basename = self.basename(version=version) 258 | 259 | if not use_cache: 260 | self.disableCache() 261 | 262 | step_number = 0 263 | while calls: 264 | step_number += 1 265 | new_commits, args, expected, expect_cache_hit = calls.pop(0) 266 | if new_commits > 0: 267 | self.fixtures.create_commits(new_commits) 268 | self.scmlogs.annotate( 269 | "step #%s: about to run tar_scm with args: %s" % 270 | (step_number, pformat(args))) 271 | self.scmlogs.annotate("expecting tar to contain: " + expected) 272 | self.tar_scm_std(*args) 273 | logpath = self.scmlogs.current_log_path 274 | loglines = self.scmlogs.read() 275 | 276 | if expect_cache_hit: 277 | self.scmlogs.annotate("expected cache hit") 278 | self.assertRanUpdate(logpath, loglines) 279 | else: 280 | self.assertRanInitialClone(logpath, loglines) 281 | 282 | if self.fixtures.subdir in args: 283 | tar_handle = self.assertTarOnly( 284 | basename, 285 | tarchecker=self.assertSubdirTar 286 | ) 287 | tarent = 'b' 288 | else: 289 | tar_handle = self.assertTarOnly(basename) 290 | tarent = 'a' 291 | 292 | self.assertTarMemberContains( 293 | tar_handle, 294 | basename + '/' + tarent, 295 | expected 296 | ) 297 | 298 | self.scmlogs.nextlog() 299 | 300 | def test_switch_revision_and_subdir(self): 301 | self._switch_revision_and_subdir() 302 | 303 | def test_switch_rev_and_subdir_nc(self): 304 | self._switch_revision_and_subdir(use_cache=False) 305 | 306 | def _switch_revision_and_subdir(self, use_cache=True): 307 | version = '5.0' 308 | args = [ 309 | '--version', version, 310 | ] 311 | args_subdir = args + ['--subdir', self.fixtures.subdir] 312 | 313 | args_tag2 = args + ['--revision', self.rev(2)] 314 | 315 | use_cache_exception = use_cache 316 | if self.scm in ('svn', 'git'): 317 | use_cache_exception = False 318 | 319 | self.scmlogs.annotate("use_cache_exception: " + str(use_cache_exception)) 320 | self._sequential_calls_with_revision( 321 | version, 322 | [ 323 | (0, args_tag2, '2', False), 324 | (0, args_subdir, '2', use_cache_exception), 325 | (2, args_tag2, '2', use_cache), 326 | (0, args_subdir, '4', use_cache), 327 | (2, args_tag2, '2', use_cache), 328 | (0, args_subdir, '6', use_cache), 329 | (0, args_tag2, '2', use_cache), 330 | ], 331 | use_cache 332 | ) 333 | 334 | def test_sslverify_disabled(self): 335 | self.tar_scm_std('--sslverify', 'disable') 336 | logpath = self.scmlogs.current_log_path 337 | loglines = self.scmlogs.read() 338 | self.assertRanInitialClone(logpath, loglines) 339 | self.assertSSLVerifyFalse(logpath, loglines) 340 | 341 | def test_sslverify_enabled(self): 342 | self.tar_scm_std('--sslverify', 'enable') 343 | logpath = self.scmlogs.current_log_path 344 | loglines = self.scmlogs.read() 345 | self.assertRanInitialClone(logpath, loglines) 346 | -------------------------------------------------------------------------------- /tests/fake_classes.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | class FakeCli(dict): # pylint: disable=no-init,too-few-public-methods 3 | url = '' 4 | revision = '' 5 | changesgenerate = False 6 | subdir = '' 7 | user = '' 8 | keyring_passphrase = '' 9 | maintainers_asc = None 10 | def __init__(self, match_tag=False): 11 | super(FakeCli, self).__init__() 12 | self.match_tag = match_tag 13 | 14 | 15 | class FakeTasks(): # pylint: disable=no-init,too-few-public-methods 16 | pass 17 | 18 | 19 | class FakeSCM(): 20 | def __init__(self, version): 21 | self.version = version 22 | self.maintainers_asc = None 23 | 24 | # pylint: disable=unused-argument,no-self-use,no-init, 25 | # pylint: disable=too-few-public-methods 26 | def detect_version(self, args): 27 | return self.version 28 | 29 | def version_iso_cleanup(self, version, debian): 30 | return self.version 31 | -------------------------------------------------------------------------------- /tests/fixtures.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # pylint: disable=R0902 3 | 4 | import os 5 | import shutil 6 | import io 7 | 8 | from utils import file_write_legacy 9 | 10 | class Fixtures: 11 | 12 | """Base class for all fixture classes.""" 13 | 14 | name = 'tar_scm test suite' 15 | email = 'root@localhost' 16 | name_and_email = '%s <%s>' % (name, email) 17 | 18 | subdir = 'subdir' 19 | subdir1 = 'subdir1' 20 | subdir2 = 'subdir2' 21 | _next_commit_revs = {} 22 | 23 | # the timestamp (in seconds since epoch ) that should be used for commits 24 | COMMITTER_DATE = int(1234567890) 25 | 26 | def __init__(self, container_dir, scmlogs): 27 | self.container_dir = container_dir 28 | self.scmlogs = scmlogs 29 | self.repo_path = self.container_dir + '/repo' 30 | self.repo_url = 'file://' + self.repo_path 31 | self.wdir = None 32 | self.user_name = None 33 | self.user_email = None 34 | self.timestamps = None 35 | self.sha1s = None 36 | self.short_sha1s = None 37 | self.wd_path = None 38 | self.added = None 39 | self.submodules_path = None 40 | 41 | 42 | # Keys are stringified integers representing commit sequence numbers; 43 | # values can be passed to --revision 44 | self.revs = {} 45 | 46 | def safe_run(self, cmd): 47 | stdout, stderr, exitcode = self.run(cmd) 48 | if exitcode != 0: 49 | raise RuntimeError("Command failed; aborting.") 50 | return stdout, stderr, exitcode 51 | 52 | def setup(self): 53 | print(self.__class__.__name__ + ": setting up fixtures") 54 | self.init_fixtures_dir() 55 | self.init() 56 | 57 | def init_fixtures_dir(self): 58 | if os.path.exists(self.repo_path): 59 | shutil.rmtree(self.repo_path) 60 | 61 | def init(self): 62 | raise NotImplementedError( 63 | self.__class__.__name__ + " didn't implement init()") 64 | 65 | def create_commits(self, num_commits, wdir=None, subdir=None): 66 | self.scmlogs.annotate("Creating %d commits ..." % num_commits) 67 | if num_commits == 0: 68 | return 69 | 70 | if wdir is None: 71 | wdir = self.wdir 72 | orig_wd = os.getcwd() 73 | os.chdir(wdir) 74 | 75 | for inc in range(0, num_commits): # pylint: disable=W0612 76 | new_rev = self.create_commit(wdir, subdir=subdir) 77 | self.record_rev(new_rev, wdir) 78 | 79 | self.scmlogs.annotate("Created %d commits; now at %s" % 80 | (num_commits, new_rev)) 81 | os.chdir(orig_wd) 82 | 83 | def next_commit_rev(self, wdir): 84 | if wdir not in self._next_commit_revs: 85 | self._next_commit_revs[wdir] = 1 86 | new_rev = self._next_commit_revs[wdir] 87 | self._next_commit_revs[wdir] += 1 88 | return new_rev 89 | 90 | def create_commit(self, wdir, subdir=None): 91 | new_rev = self.next_commit_rev(wdir) 92 | newly_created = self.prep_commit(new_rev, subdir=subdir) 93 | self.do_commit(wdir, new_rev, newly_created) 94 | return new_rev 95 | 96 | def do_commit(self, wdir, new_rev, newly_created): # pylint: disable=W0613 97 | self.safe_run('add .') 98 | date = self.get_committer_date() 99 | self.safe_run('commit -m%d %s' % (new_rev, date)) 100 | 101 | def get_committer_date(self): 102 | return '--date="%s"' % str(self.COMMITTER_DATE) 103 | 104 | def prep_commit(self, new_rev, subdir=None): 105 | """ 106 | Caller should ensure correct cwd. 107 | Returns list of newly created files. 108 | """ 109 | if not subdir: 110 | subdir = self.subdir 111 | self.scmlogs.annotate("cwd is %s" % os.getcwd()) 112 | newly_created = [] 113 | 114 | if not os.path.exists('a'): 115 | newly_created.append('a') 116 | 117 | if not os.path.exists(subdir): 118 | os.mkdir(subdir) 119 | # This will take care of adding subdir/b too 120 | newly_created.append(subdir) 121 | 122 | for fname in ('a', subdir + '/b'): 123 | file_write_legacy(fname, new_rev) 124 | self.scmlogs.annotate("Wrote %s to %s" % (new_rev, fname)) 125 | 126 | # we never commit through symlink 'c' but instead see the updated 127 | # revision through the symlink 128 | if not os.path.lexists('c'): 129 | os.symlink('a', 'c') 130 | newly_created.append('c') 131 | 132 | return newly_created 133 | 134 | def create_commit_broken_symlink(self, wdir=None): 135 | self.scmlogs.annotate("Creating broken symlink commit") 136 | 137 | if wdir is None: 138 | wdir = self.wdir 139 | os.chdir(wdir) 140 | 141 | new_rev = self.next_commit_rev(wdir) 142 | newly_created = self.prep_commit(new_rev) 143 | os.unlink('c') 144 | os.symlink('/../nir/va/na', 'c') 145 | newly_created.append('c') 146 | self.do_commit(wdir, new_rev, newly_created) 147 | self.record_rev(new_rev, wdir) 148 | self.scmlogs.annotate("Created 1 commit; now at %s" % (new_rev)) 149 | 150 | def create_commit_unicode(self, wdir=None): 151 | self.scmlogs.annotate("Creating commit with unicode commit message") 152 | 153 | if wdir is None: 154 | wdir = self.wdir 155 | os.chdir(wdir) 156 | 157 | new_rev = self.next_commit_rev(wdir) 158 | fname = 'd' 159 | file_write_legacy(fname, new_rev) 160 | self.scmlogs.annotate("Wrote %s to %s" % (new_rev, fname)) 161 | self.safe_run('add .') 162 | date = self.get_committer_date() 163 | self.safe_run('commit -m"füfüfü nününü %d" %s' % (new_rev, date)) 164 | self.record_rev(new_rev, wdir) 165 | self.scmlogs.annotate("Created 1 commit; now at %s" % (new_rev)) 166 | 167 | def touch(self, fname, times=None): 168 | fpath = os.path.join(self.wdir, fname) 169 | with io.open(fpath, 'a', encoding='UTF-8'): 170 | os.utime(fname, times) 171 | 172 | def remove(self, fname): 173 | os.remove(os.path.join(self.wdir, fname)) 174 | 175 | def tag(self, tag): 176 | self.safe_run('tag %s' % tag) 177 | 178 | def commit_file_with_tag(self, tag, file): 179 | self.touch(file) 180 | self.create_commit(self.wdir) 181 | self.tag(tag) 182 | -------------------------------------------------------------------------------- /tests/fixtures/ArchiveOBSCpioTestCases/test_obscpio_broken_link/repo/broken_link: -------------------------------------------------------------------------------- 1 | non-existant-file -------------------------------------------------------------------------------- /tests/fixtures/ArchiveOBSCpioTestCases/test_obscpio_create_archive/repo/Readme.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openSUSE/obs-service-tar_scm/bfbf96f2b1dcd8e1c0743ec69e5c142807252981/tests/fixtures/ArchiveOBSCpioTestCases/test_obscpio_create_archive/repo/Readme.md -------------------------------------------------------------------------------- /tests/fixtures/ArchiveOBSCpioTestCases/test_obscpio_create_archive/repo/a: -------------------------------------------------------------------------------- 1 | a 2 | -------------------------------------------------------------------------------- /tests/fixtures/ArchiveOBSCpioTestCases/test_obscpio_create_archive/repo/dir1/.keep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openSUSE/obs-service-tar_scm/bfbf96f2b1dcd8e1c0743ec69e5c142807252981/tests/fixtures/ArchiveOBSCpioTestCases/test_obscpio_create_archive/repo/dir1/.keep -------------------------------------------------------------------------------- /tests/fixtures/ArchiveOBSCpioTestCases/test_obscpio_create_archive/repo/test.spec: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openSUSE/obs-service-tar_scm/bfbf96f2b1dcd8e1c0743ec69e5c142807252981/tests/fixtures/ArchiveOBSCpioTestCases/test_obscpio_create_archive/repo/test.spec -------------------------------------------------------------------------------- /tests/fixtures/ArchiveOBSCpioTestCases/test_obscpio_extract_d/repo/Readme.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openSUSE/obs-service-tar_scm/bfbf96f2b1dcd8e1c0743ec69e5c142807252981/tests/fixtures/ArchiveOBSCpioTestCases/test_obscpio_extract_d/repo/Readme.md -------------------------------------------------------------------------------- /tests/fixtures/ArchiveOBSCpioTestCases/test_obscpio_extract_d/repo/a: -------------------------------------------------------------------------------- 1 | a 2 | -------------------------------------------------------------------------------- /tests/fixtures/ArchiveOBSCpioTestCases/test_obscpio_extract_d/repo/dir1/.keep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openSUSE/obs-service-tar_scm/bfbf96f2b1dcd8e1c0743ec69e5c142807252981/tests/fixtures/ArchiveOBSCpioTestCases/test_obscpio_extract_d/repo/dir1/.keep -------------------------------------------------------------------------------- /tests/fixtures/ArchiveOBSCpioTestCases/test_obscpio_extract_d/repo/test.spec: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openSUSE/obs-service-tar_scm/bfbf96f2b1dcd8e1c0743ec69e5c142807252981/tests/fixtures/ArchiveOBSCpioTestCases/test_obscpio_extract_d/repo/test.spec -------------------------------------------------------------------------------- /tests/fixtures/ArchiveOBSCpioTestCases/test_obscpio_extract_glob/repo/Readme.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openSUSE/obs-service-tar_scm/bfbf96f2b1dcd8e1c0743ec69e5c142807252981/tests/fixtures/ArchiveOBSCpioTestCases/test_obscpio_extract_glob/repo/Readme.md -------------------------------------------------------------------------------- /tests/fixtures/ArchiveOBSCpioTestCases/test_obscpio_extract_glob/repo/a: -------------------------------------------------------------------------------- 1 | a 2 | -------------------------------------------------------------------------------- /tests/fixtures/ArchiveOBSCpioTestCases/test_obscpio_extract_glob/repo/dir1/.keep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openSUSE/obs-service-tar_scm/bfbf96f2b1dcd8e1c0743ec69e5c142807252981/tests/fixtures/ArchiveOBSCpioTestCases/test_obscpio_extract_glob/repo/dir1/.keep -------------------------------------------------------------------------------- /tests/fixtures/ArchiveOBSCpioTestCases/test_obscpio_extract_glob/repo/test.rpmlintrc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openSUSE/obs-service-tar_scm/bfbf96f2b1dcd8e1c0743ec69e5c142807252981/tests/fixtures/ArchiveOBSCpioTestCases/test_obscpio_extract_glob/repo/test.rpmlintrc -------------------------------------------------------------------------------- /tests/fixtures/ArchiveOBSCpioTestCases/test_obscpio_extract_glob/repo/test.spec: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openSUSE/obs-service-tar_scm/bfbf96f2b1dcd8e1c0743ec69e5c142807252981/tests/fixtures/ArchiveOBSCpioTestCases/test_obscpio_extract_glob/repo/test.spec -------------------------------------------------------------------------------- /tests/fixtures/ArchiveOBSCpioTestCases/test_obscpio_extract_mf/repo/Readme.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openSUSE/obs-service-tar_scm/bfbf96f2b1dcd8e1c0743ec69e5c142807252981/tests/fixtures/ArchiveOBSCpioTestCases/test_obscpio_extract_mf/repo/Readme.md -------------------------------------------------------------------------------- /tests/fixtures/ArchiveOBSCpioTestCases/test_obscpio_extract_mf/repo/a: -------------------------------------------------------------------------------- 1 | a 2 | -------------------------------------------------------------------------------- /tests/fixtures/ArchiveOBSCpioTestCases/test_obscpio_extract_mf/repo/dir1/.keep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openSUSE/obs-service-tar_scm/bfbf96f2b1dcd8e1c0743ec69e5c142807252981/tests/fixtures/ArchiveOBSCpioTestCases/test_obscpio_extract_mf/repo/dir1/.keep -------------------------------------------------------------------------------- /tests/fixtures/ArchiveOBSCpioTestCases/test_obscpio_extract_mf/repo/test.spec: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openSUSE/obs-service-tar_scm/bfbf96f2b1dcd8e1c0743ec69e5c142807252981/tests/fixtures/ArchiveOBSCpioTestCases/test_obscpio_extract_mf/repo/test.spec -------------------------------------------------------------------------------- /tests/fixtures/ArchiveOBSCpioTestCases/test_obscpio_extract_of/repo/Readme.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openSUSE/obs-service-tar_scm/bfbf96f2b1dcd8e1c0743ec69e5c142807252981/tests/fixtures/ArchiveOBSCpioTestCases/test_obscpio_extract_of/repo/Readme.md -------------------------------------------------------------------------------- /tests/fixtures/ArchiveOBSCpioTestCases/test_obscpio_extract_of/repo/a: -------------------------------------------------------------------------------- 1 | a 2 | -------------------------------------------------------------------------------- /tests/fixtures/ArchiveOBSCpioTestCases/test_obscpio_extract_of/repo/dir1/.keep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openSUSE/obs-service-tar_scm/bfbf96f2b1dcd8e1c0743ec69e5c142807252981/tests/fixtures/ArchiveOBSCpioTestCases/test_obscpio_extract_of/repo/dir1/.keep -------------------------------------------------------------------------------- /tests/fixtures/ArchiveOBSCpioTestCases/test_obscpio_extract_of/repo/test.spec: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openSUSE/obs-service-tar_scm/bfbf96f2b1dcd8e1c0743ec69e5c142807252981/tests/fixtures/ArchiveOBSCpioTestCases/test_obscpio_extract_of/repo/test.spec -------------------------------------------------------------------------------- /tests/fixtures/GitTests/test_find_valid_commit/fixtures.tar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openSUSE/obs-service-tar_scm/bfbf96f2b1dcd8e1c0743ec69e5c142807252981/tests/fixtures/GitTests/test_find_valid_commit/fixtures.tar -------------------------------------------------------------------------------- /tests/fixtures/TasksTestCases/test_appimage_empty_build/appimage.yml: -------------------------------------------------------------------------------- 1 | app: APPIMAGE_NAME 2 | binpatch: true 3 | 4 | ingredients: 5 | packages: 6 | - RPM_PACKAGE_NAME 7 | 8 | script: 9 | - cd $BUILD_APPDIR/ 10 | - cp $BUILD_APPDIR/usr/share/applications/NAME.desktop $BUILD_APPDIR 11 | - cp $BUILD_APPDIR/usr/share/pixmaps/NAME.png $BUILD_APPDIR 12 | -------------------------------------------------------------------------------- /tests/fixtures/TasksTestCases/test_appimage_empty_build_git/appimage.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openSUSE/obs-service-tar_scm/bfbf96f2b1dcd8e1c0743ec69e5c142807252981/tests/fixtures/TasksTestCases/test_appimage_empty_build_git/appimage.yml -------------------------------------------------------------------------------- /tests/fixtures/TasksTestCases/test_generate_tl_multi_tasks/snapcraft.yaml: -------------------------------------------------------------------------------- 1 | name: pipelinetest 2 | version: 1.0 3 | summary: Libpipeline example 4 | description: | 5 | This is an example package of an autotools project built with snapcraft 6 | using a remote source. 7 | 8 | apps: 9 | pipelinetest: 10 | command: ./bin/test 11 | 12 | parts: 13 | kanku: 14 | plugin: make 15 | source: git@github.com:M0ses/kanku 16 | source-type: git 17 | after: 18 | - libpipeline 19 | libpipeline: 20 | plugin: autotools 21 | source: lp:~mterry/libpipeline/printf 22 | source-type: bzr 23 | 24 | -------------------------------------------------------------------------------- /tests/fixtures/TasksTestCases/test_generate_tl_single_task/snapcraft.yaml: -------------------------------------------------------------------------------- 1 | name: pipelinetest 2 | version: 1.0 3 | summary: Libpipeline example 4 | description: | 5 | This is an example package of an autotools project built with snapcraft 6 | using a remote source. 7 | 8 | apps: 9 | pipelinetest: 10 | command: ./bin/test 11 | 12 | parts: 13 | pipelinetest: 14 | plugin: make 15 | source: . 16 | after: 17 | - libpipeline 18 | unkown-source-type: 19 | plugin: make 20 | source: . 21 | source-type: not-known 22 | after: 23 | - libpipeline 24 | libpipeline: 25 | plugin: autotools 26 | source: lp:~mterry/libpipeline/printf 27 | source-type: bzr 28 | 29 | -------------------------------------------------------------------------------- /tests/fixtures/TasksTestCases/test_generate_tl_st_appimage/appimage.yml: -------------------------------------------------------------------------------- 1 | app: QtQuickApp 2 | 3 | build: 4 | packages: 5 | - linuxdeployqt 6 | - pkgconfig(Qt5Quick) 7 | git: 8 | - https://github.com/probonopd/QtQuickApp.git 9 | 10 | -------------------------------------------------------------------------------- /tests/fixtures/TasksTestCases/test_tasks_finalize: -------------------------------------------------------------------------------- 1 | test_generate_tl_multi_tasks/ -------------------------------------------------------------------------------- /tests/fixtures/UnitTestCases/test_config_debug_tar_scm/test.rc: -------------------------------------------------------------------------------- 1 | var=var 2 | -------------------------------------------------------------------------------- /tests/fixtures/UnitTestCases/test_config_files_ordering/a.cfg: -------------------------------------------------------------------------------- 1 | var=a 2 | -------------------------------------------------------------------------------- /tests/fixtures/UnitTestCases/test_config_files_ordering/b.cfg: -------------------------------------------------------------------------------- 1 | var=b 2 | -------------------------------------------------------------------------------- /tests/fixtures/UnitTestCases/test_config_no_faked_header/test.ini: -------------------------------------------------------------------------------- 1 | [general] 2 | apiurl = http://api.example.com 3 | 4 | [http://api.example.com] 5 | email = devel@example.com 6 | -------------------------------------------------------------------------------- /tests/fixtures/UnitTestCases/test_unicode_in_filename/test/äöü.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openSUSE/obs-service-tar_scm/bfbf96f2b1dcd8e1c0743ec69e5c142807252981/tests/fixtures/UnitTestCases/test_unicode_in_filename/test/äöü.txt -------------------------------------------------------------------------------- /tests/gitfixtures.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import os 4 | 5 | from fixtures import Fixtures 6 | from utils import run_git 7 | 8 | 9 | class GitFixtures(Fixtures): 10 | 11 | """Methods to create and populate a git repository. 12 | 13 | git tests use this class in order to have something to test against. 14 | """ 15 | 16 | def init(self): 17 | self.user_name = 'test' 18 | self.user_email = 'test@test.com' 19 | 20 | tmpdir = os.path.join(os.path.dirname( 21 | os.path.abspath(__file__)), 'tmp') 22 | gitconfig = os.path.join(tmpdir, '.gitconfig') 23 | os.environ["GIT_CONFIG_GLOBAL"] = gitconfig 24 | self.safe_run('config --global protocol.file.allow always') 25 | self.safe_run('config --global commit.gpgsign false') 26 | 27 | self.create_repo(self.repo_path) 28 | self.wdir = self.repo_path 29 | self.submodules_path = self.container_dir + '/submodules' 30 | 31 | # These will be two-level dicts; top level keys are 32 | # repo paths (this allows us to track the main repo 33 | # *and* submodules). 34 | self.timestamps = {} 35 | self.sha1s = {} 36 | 37 | # Force the committer timestamp to our well known default 38 | os.environ["GIT_COMMITTER_DATE"] = self.get_committer_date() 39 | 40 | self.create_commits(2) 41 | 42 | def run(self, cmd): # pylint: disable=R0201 43 | return run_git(cmd) 44 | 45 | def create_repo(self, repo_path): 46 | os.makedirs(repo_path) 47 | os.chdir(repo_path) 48 | self.safe_run('init') 49 | self.safe_run('config user.name ' + self.user_name) 50 | self.safe_run('config user.email ' + self.user_email) 51 | print("created repo %s" % repo_path) 52 | 53 | def get_metadata(self, fmt): 54 | return self.safe_run('log -n1 --pretty=format:"%s"' % fmt)[0].decode() 55 | 56 | def record_rev(self, rev_num, *args): 57 | wdir = args[0] 58 | print(" ****** wdir: %s" % wdir) 59 | tag = 'tag' + str(rev_num) 60 | self.safe_run('tag ' + tag) 61 | 62 | for dname in (self.revs, self.timestamps, self.sha1s): 63 | if wdir not in dname: 64 | dname[wdir] = {} 65 | 66 | self.revs[wdir][rev_num] = tag 67 | self.timestamps[wdir][tag] = self.get_metadata('%ct') 68 | self.sha1s[wdir][tag] = self.get_metadata('%H') 69 | self.scmlogs.annotate( 70 | "Recorded rev %d: id %s, timestamp %s, SHA1 %s in %s" % 71 | (rev_num, 72 | tag, 73 | self.timestamps[wdir][tag], 74 | self.sha1s[wdir][tag], 75 | wdir) 76 | ) 77 | 78 | def submodule_path(self, submodule_name): 79 | return self.submodules_path + '/' + submodule_name 80 | 81 | def create_submodule(self, submodule_name): 82 | path = self.submodule_path(submodule_name) 83 | # self.scmlogs.annotate("Creating repo in %s" % path) 84 | self.create_repo(path) 85 | -------------------------------------------------------------------------------- /tests/githgtests.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # pylint: disable=E1101 4 | from commontests import CommonTests 5 | 6 | 7 | class GitHgTests(CommonTests): 8 | 9 | """Unit tests which are shared between git and hg.""" 10 | 11 | mixed_version_template = '%s.master.%s' 12 | 13 | def test_versionformat_abbrevhash(self): 14 | self.tar_scm_std('--versionformat', self.abbrev_hash_format) 15 | self.assertTarOnly( 16 | self.basename(version=self.abbrev_sha1s(self.rev(2)))) 17 | 18 | def test_versionformat_timestamp(self): 19 | self.tar_scm_std('--versionformat', self.timestamp_format) 20 | self.assertTarOnly( 21 | self.basename(version=self.version(2))) 22 | 23 | def test_versionformat_dateYYYYMMDD(self): # pylint: disable=C0103 24 | self.tar_scm_std('--versionformat', self.yyyymmdd_format) 25 | self.assertTarOnly( 26 | self.basename(version=self.dateYYYYMMDD(self.rev(2)))) 27 | 28 | def test_versionformat_dateYYYYMMDDHHMMSS(self): # pylint: disable=C0103 29 | self.tar_scm_std('--versionformat', self.yyyymmddhhmmss_format) 30 | ver = self.dateYYYYMMDDHHMMSS(self.rev(2)) 31 | print(ver) 32 | self.assertTarOnly(self.basename(version=ver)) 33 | 34 | def _mixed_version_format(self): 35 | return self.mixed_version_template % \ 36 | (self.timestamp_format, self.abbrev_hash_format) 37 | 38 | def _mixed_version(self, rev): 39 | return self.mixed_version_template % \ 40 | (self.version(rev), self.abbrev_sha1s(self.rev(rev))) 41 | 42 | def test_versionformat_mixed(self): 43 | self.tar_scm_std('--versionformat', self._mixed_version_format()) 44 | self.assertTarOnly(self.basename(version=self._mixed_version(2))) 45 | 46 | def test_version_versionformat(self): 47 | self.tar_scm_std('--version', '3.0', 48 | '--versionformat', self._mixed_version_format()) 49 | self.assertTarOnly(self.basename(version=self._mixed_version(2))) 50 | 51 | def test_versionformat_revision(self): 52 | self.fixtures.create_commits(4) 53 | self.tar_scm_std('--versionformat', self.abbrev_hash_format, 54 | '--revision', self.rev(2)) 55 | basename = self.basename(version=self.abbrev_sha1s(self.rev(2))) 56 | tar = self.assertTarOnly(basename) 57 | self.assertTarMemberContains(tar, basename + '/a', '2') 58 | -------------------------------------------------------------------------------- /tests/gitsvntests.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # pylint: disable=W1401,E1101 3 | # noqa: W605,E501 4 | import os 5 | import textwrap 6 | import re 7 | import shutil 8 | import glob 9 | import six 10 | 11 | from commontests import CommonTests 12 | 13 | 14 | class GitSvnTests(CommonTests): 15 | 16 | """Unit tests which are shared between git and svn.""" 17 | 18 | def _tar_scm_changesgenerate(self, mode, **kwargs): 19 | self.tar_scm_std( 20 | '--changesauthor', 'a@b.c', 21 | '--changesgenerate', mode, 22 | **kwargs 23 | ) 24 | 25 | def test_changesgenerate_disabled(self): 26 | self._tar_scm_changesgenerate('disable') 27 | 28 | def test_changesgenerate_no_servicedata(self): # pylint: disable=C0103 29 | self._tar_scm_changesgenerate('enable') 30 | self._check_servicedata() 31 | 32 | def test_changesgenerate_corrupt_servicedata(self): # pylint: disable=C0103 33 | with open(os.path.join(self.pkgdir, '_servicedata'), 'w') as sdat: 34 | sdat.write('this is not valid xml') 35 | self._tar_scm_changesgenerate('enable', should_succeed=False) 36 | 37 | def test_changesgenerate_empty_servicedata_file(self): # pylint: disable=C0103 38 | sdat = open(os.path.join(self.pkgdir, '_servicedata'), 'w') 39 | sdat.close() 40 | self._tar_scm_changesgenerate('enable') 41 | self._check_servicedata() 42 | 43 | def test_changesgenerate_empty_servicedata_element(self): # pylint: disable=C0103 44 | with open(os.path.join(self.pkgdir, '_servicedata'), 'w') as sdat: 45 | sdat.write("\n\n") 46 | self._tar_scm_changesgenerate('enable') 47 | self._check_servicedata() 48 | 49 | def test_changesgenerate_no_changesrevision(self): # pylint: disable=C0103 50 | with open(os.path.join(self.pkgdir, '_servicedata'), 'w') as sdat: 51 | sdat.write(textwrap.dedent("""\ 52 | 53 | 54 | %s 55 | 56 | """ % self.fixtures.repo_url)) 57 | self._tar_scm_changesgenerate('enable') 58 | self._check_servicedata() 59 | 60 | def _write_changes_file(self): 61 | contents = textwrap.dedent("""\ 62 | ------------------------------------------------------------------- 63 | Fri Oct 3 00:17:50 BST 2014 - %s 64 | 65 | - 2 66 | 67 | ------------------------------------------------------------------- 68 | Thu Sep 18 10:27:14 BST 2014 - %s 69 | 70 | - 1 71 | """ % (self.fixtures.user_email, self.fixtures.user_email)) 72 | with open(os.path.join(self.pkgdir, 'pkg.changes'), 'w') as pkg: 73 | pkg.write(contents) 74 | return contents 75 | 76 | def test_changesgenerate_no_change_or_changes_file(self): # pylint: disable=C0103 77 | self._write_servicedata(2) 78 | self._tar_scm_changesgenerate('enable') 79 | self._check_servicedata() 80 | 81 | def test_changesgenerate_no_change_same_changes_file(self): # pylint: disable=C0103 82 | self._write_servicedata(2) 83 | self._write_changes_file() 84 | self._tar_scm_changesgenerate('enable') 85 | self._check_servicedata() 86 | 87 | def test_changesgenerate_new_commit_no_changes_file(self): # pylint: disable=C0103 88 | self._write_servicedata(2) 89 | self.fixtures.create_commits(1) 90 | self._tar_scm_changesgenerate('enable') 91 | self._check_servicedata(revision=3) 92 | 93 | def _new_change_entry_regexp(self, author, changes): # pylint: disable=R0201 94 | return textwrap.dedent("""\ 95 | ^------------------------------------------------------------------- 96 | \w{3} \w{3} [ \d]\d \d\d:\d\d:\d\d [A-Z]{3} 20\d\d - %s 97 | 98 | %s 99 | """) % (author, changes) 100 | 101 | def _check_changes(self, orig_changes, expected_changes_regexp): 102 | new_changes_file = os.path.join(self.outdir, 'pkg.changes') 103 | self.assertTrue(os.path.exists(new_changes_file)) 104 | with open(new_changes_file) as chg: 105 | new_changes = chg.read() 106 | self.assertNotEqual(orig_changes, new_changes) 107 | print(new_changes) 108 | expected_changes_regexp += "(.*)" 109 | six.assertRegex(self, new_changes, expected_changes_regexp) 110 | reg = re.match(expected_changes_regexp, new_changes, re.DOTALL) 111 | self.assertEqual(reg.group(1), orig_changes) 112 | 113 | def test_changesgenerate_new_commit_and_changes_file(self): # pylint: disable=C0103 114 | self._test_changesgenerate_new_commit_and_changes_file( 115 | self.fixtures.user_email, self.fixtures.user_email) 116 | 117 | def test_changesgenerate_new_commit_and_changes_file_default_author(self): # pylint: disable=C0103 118 | os.environ['OBS_SERVICE_DAEMON'] = "1" 119 | self._test_changesgenerate_new_commit_and_changes_file(None, 'obs-service-tar-scm@invalid') 120 | os.environ['OBS_SERVICE_DAEMON'] = "0" 121 | 122 | def test_changesgenerate_new_commit_and_changes_file_full_author(self): # pylint: disable=C0103 123 | os.environ['OBS_SERVICE_DAEMON'] = "1" 124 | os.environ['VC_REALNAME'] = 'Tar Scm Service' 125 | os.environ['VC_MAILADDR'] = 'obs-service-tar-scm@invalid' 126 | self._test_changesgenerate_new_commit_and_changes_file(None, 'Tar Scm Service ') 127 | os.environ['OBS_SERVICE_DAEMON'] = "0" 128 | del os.environ['VC_REALNAME'] 129 | del os.environ['VC_MAILADDR'] 130 | 131 | def _write_servicedata(self, rev): 132 | with open(os.path.join(self.pkgdir, '_servicedata'), 'w') as sdat: 133 | sdat.write(textwrap.dedent("""\ 134 | 135 | 136 | %s 137 | %s 138 | 139 | """ % (self.fixtures.repo_url, self.changesrevision(rev)))) 140 | 141 | def _test_changesgenerate_new_commit_and_changes_file(self, changesauthor, expected_author): # pylint: disable=C0103 142 | self._write_servicedata(2) 143 | orig_changes = self._write_changes_file() 144 | self.fixtures.create_commits(3) 145 | rev = 5 146 | print("XXXX 1") 147 | tar_scm_args = self.tar_scm_args() 148 | 149 | if changesauthor is not None: 150 | tar_scm_args += ['--changesauthor', changesauthor] 151 | 152 | print("XXXX 2") 153 | self.tar_scm_std(*tar_scm_args) 154 | 155 | print("XXXX 3") 156 | self._check_servicedata(revision=rev, expected_dirents=3) 157 | 158 | rev = self.changesrevision(rev, abbrev=True) 159 | 160 | print("XXXX 4") 161 | expected_changes_regexp = self._new_change_entry_regexp( 162 | expected_author, 163 | textwrap.dedent("""\ 164 | - Update to version 0.6.%s: 165 | \* 5 166 | \* 4 167 | \* 3 168 | """) % rev 169 | ) 170 | self._check_changes(orig_changes, expected_changes_regexp) 171 | 172 | def test_changesgenerate_new_commit_and_changes_file_no_version(self): # pylint: disable=C0103 173 | self._write_servicedata(2) 174 | orig_changes = self._write_changes_file() 175 | self.fixtures.create_commits(3) 176 | rev = 5 177 | 178 | tar_scm_args = [ 179 | '--changesgenerate', 'enable', 180 | '--version', '', 181 | '--changesauthor', self.fixtures.user_email 182 | ] 183 | self.tar_scm_std(*tar_scm_args) 184 | 185 | self._check_servicedata(revision=rev, expected_dirents=3) 186 | 187 | rev = self.changesrevision(rev, abbrev=True) 188 | ver_regex = self.changesregex(rev) 189 | 190 | expected_author = self.fixtures.user_email 191 | expected_changes_regexp = self._new_change_entry_regexp( 192 | expected_author, 193 | textwrap.dedent("""\ 194 | - Update to version %s: 195 | \* 5 196 | \* 4 197 | \* 3 198 | """) % ver_regex 199 | ) 200 | self._check_changes(orig_changes, expected_changes_regexp) 201 | 202 | def test_changesgenerate_new_commit_and_changes_file_with_subdir(self): # pylint: disable=C0103 203 | self._write_servicedata(2) 204 | orig_changes = self._write_changes_file() 205 | self.fixtures.create_commits(3) 206 | self.fixtures.create_commits(3, subdir='another_subdir') 207 | rev = 8 208 | 209 | tar_scm_args = self.tar_scm_args() 210 | 211 | tar_scm_args += [ 212 | '--subdir', 'another_subdir', 213 | '--changesauthor', self.fixtures.user_email, 214 | ] 215 | 216 | self.tar_scm_std(*tar_scm_args) 217 | 218 | self._check_servicedata(revision=rev, expected_dirents=3) 219 | 220 | expected_author = self.fixtures.user_email 221 | expected_changes_regexp = self._new_change_entry_regexp( 222 | expected_author, 223 | textwrap.dedent("""\ 224 | - Update to version 0.6.%s: 225 | \* 8 226 | \* 7 227 | \* 6 228 | """) % self.changesrevision(rev, abbrev=True) 229 | ) 230 | self._check_changes(orig_changes, expected_changes_regexp) 231 | 232 | def test_changesgenerate_old_servicedata(self): # pylint: disable=C0103 233 | self._write_servicedata(2) 234 | orig_changes = self._write_changes_file() 235 | self.fixtures.create_commits(3) 236 | rev = 5 237 | sd_file = os.path.join(self.pkgdir, '_servicedata') 238 | old_dir = os.path.join(self.pkgdir, '.old') 239 | os.mkdir(old_dir) 240 | shutil.move(sd_file, old_dir) 241 | for filename in glob.glob(os.path.join(self.pkgdir, '*.changes')): 242 | shutil.move(filename, old_dir) 243 | 244 | tar_scm_args = self.tar_scm_args() 245 | 246 | tar_scm_args += [ 247 | '--changesauthor', self.fixtures.user_email, 248 | ] 249 | 250 | self.tar_scm_std(*tar_scm_args) 251 | 252 | self._check_servicedata(revision=rev, expected_dirents=3) 253 | 254 | expected_author = self.fixtures.user_email 255 | expected_changes_regexp = self._new_change_entry_regexp( 256 | expected_author, 257 | textwrap.dedent("""\ 258 | - Update to version 0.6.%s: 259 | \* 5 260 | \* 4 261 | \* 3 262 | """) % self.changesrevision(rev, abbrev=True) 263 | ) 264 | self._check_changes(orig_changes, expected_changes_regexp) 265 | -------------------------------------------------------------------------------- /tests/hgfixtures.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import os 4 | 5 | from fixtures import Fixtures 6 | from utils import run_hg, file_write_legacy 7 | 8 | 9 | class HgFixtures(Fixtures): 10 | 11 | """Methods to create and populate a mercurial repository. 12 | 13 | mercurial tests use this class in order to have something to test against. 14 | """ 15 | 16 | def init(self): 17 | self.create_repo() 18 | 19 | self.timestamps = {} 20 | self.sha1s = {} 21 | self.short_sha1s = {} 22 | 23 | self.create_commits(2) 24 | 25 | def run(self, cmd): 26 | return run_hg(self.repo_path, cmd) 27 | 28 | def create_repo(self): 29 | os.makedirs(self.repo_path) 30 | os.chdir(self.repo_path) 31 | self.safe_run('init') 32 | out = "[ui]\nusername = %s\n" % self.name_and_email 33 | file_write_legacy('.hg/hgrc', out) 34 | 35 | self.wdir = self.repo_path 36 | print("created repo %s" % self.repo_path) 37 | 38 | def get_metadata(self, formatstr): 39 | return self.safe_run('log -l1 --template "%s"' % formatstr)[0].decode() 40 | 41 | def record_rev(self, *args): 42 | rev_num = args[0] 43 | tag = str(rev_num - 1) # hg starts counting changesets at 0 44 | self.revs[rev_num] = tag 45 | epoch_secs, tz_delta_to_utc = \ 46 | self.get_metadata('{date|hgdate}').split() 47 | self.timestamps[tag] = (float(epoch_secs), int(tz_delta_to_utc)) 48 | self.sha1s[tag] = self.get_metadata('{node}') 49 | self.short_sha1s[tag] = self.get_metadata('{node|short}') 50 | self.scmlogs.annotate( 51 | "Recorded rev %d: id %s, timestamp %s, SHA1 %s" % 52 | (rev_num, 53 | tag, 54 | self.timestamps[tag], 55 | self.sha1s[tag]) 56 | ) 57 | 58 | def get_committer_date(self): 59 | return '--date="%s"' % (str(self.COMMITTER_DATE) + " 0") 60 | -------------------------------------------------------------------------------- /tests/hgtests.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from datetime import datetime 4 | import time 5 | 6 | from githgtests import GitHgTests 7 | from hgfixtures import HgFixtures 8 | from utils import run_hg 9 | 10 | 11 | class HgTests(GitHgTests): 12 | 13 | """Unit tests for 'tar_scm --scm hg'. 14 | 15 | hg-specific tests are in this class. Other shared tests are 16 | included via the class inheritance hierarchy. 17 | """ 18 | 19 | scm = 'hg' 20 | initial_clone_command = 'hg clone' 21 | update_cache_command = 'hg pull' 22 | sslverify_false_args = '--insecure' 23 | fixtures_class = HgFixtures 24 | 25 | abbrev_hash_format = '{node|short}' 26 | timestamp_format = '{date}' 27 | yyyymmdd_format = '{date|localdate|shortdate}' 28 | yyyymmddhhmmss_format = '{date|localdate|isodatesec}' 29 | 30 | def default_version(self): 31 | return self.rev(2) 32 | 33 | def sha1s(self, rev): 34 | return self.fixtures.sha1s[rev] 35 | 36 | def abbrev_sha1s(self, rev): 37 | return self.fixtures.short_sha1s[rev] 38 | 39 | def version(self, rev): 40 | # Hyphens aren't allowed in version number. This substitution 41 | # mirrors the use of sed "s@-@@g" in tar_scm. 42 | version = "%s%s" % self.timestamps(self.rev(rev)) 43 | return version.replace('-', '') 44 | 45 | def current_utc_offset(self): 46 | now = time.time() 47 | offset = (datetime.fromtimestamp(now) - 48 | datetime.utcfromtimestamp(now)) 49 | # since total_seconds() isn't available in python 2.6 ... 50 | return ((((offset.days * 24 * 3600) + offset.seconds) * 10 ** 6) + 51 | offset.microseconds + 0.0) / 10 ** 6 52 | 53 | def dateYYYYMMDD(self, rev): 54 | # mercurial has a bug in the localdate filter that makes it apply 55 | # the current offset from UTC to historic timestamps for timezones 56 | # that have daylight savings enabled 57 | dateobj = datetime.utcfromtimestamp(self.timestamps(rev)[0] + 58 | self.current_utc_offset()) 59 | return dateobj.strftime("%4Y%02m%02d") 60 | 61 | def dateYYYYMMDDHHMMSS(self, rev): 62 | dateobj = datetime.utcfromtimestamp(self.timestamps(rev)[0] + 63 | self.current_utc_offset()) 64 | return dateobj.strftime("%4Y%02m%02dT%02H%02M%02S") 65 | 66 | def test_fetch_upstream(self): 67 | """Checkout an url that ends with a trailing slash""" 68 | repo_url = self.fixtures.repo_url + '/' 69 | args = ['--url', repo_url, '--scm', self.scm] 70 | self.tar_scm(args) 71 | -------------------------------------------------------------------------------- /tests/scm-wrapper: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Wrapper around SCM to enable behaviour verification testing 4 | # on tar_scm's repository caching code. This is cleaner than 5 | # writing tests which look inside the cache, because then they 6 | # become coupled to the cache's implementation, and require 7 | # knowledge of where the cache lives etc. 8 | 9 | me=`basename $0` 10 | 11 | if [ -z "$SCM_INVOCATION_LOG" ]; then 12 | cat <&2 13 | \$SCM_INVOCATION_LOG must be set before calling $0. 14 | It should be invoked from the test suite, not directly. 15 | EOF 16 | exit 1 17 | fi 18 | 19 | if [ "$me" = 'scm-wrapper' ]; then 20 | echo "$me should not be invoked directly, only via symlink" >&2 21 | exit 1 22 | fi 23 | 24 | echo "$me $*" >> "$SCM_INVOCATION_LOG" 25 | 26 | if [ -x /usr/bin/$me ];then 27 | /usr/bin/$me "$@" 28 | else 29 | /bin/$me "$@" 30 | fi 31 | -------------------------------------------------------------------------------- /tests/scm.py: -------------------------------------------------------------------------------- 1 | import os 2 | import shutil 3 | import unittest 4 | import six 5 | 6 | from TarSCM.scm.base import Scm 7 | 8 | import TarSCM 9 | 10 | 11 | class SCMBaseTestCases(unittest.TestCase): 12 | def setUp(self): 13 | self.basedir = os.path.abspath(os.path.dirname(__file__)) 14 | # os.getcwd() 15 | self.tests_dir = os.path.abspath(os.path.dirname(__file__)) 16 | self.tmp_dir = os.path.join(self.tests_dir, 'tmp') 17 | self.outdir = os.path.join(self.tmp_dir, 18 | self.__class__.__name__, 'out') 19 | self._prepare_cli() 20 | 21 | def tearDown(self): 22 | if os.path.exists(self.outdir): 23 | shutil.rmtree(self.outdir) 24 | 25 | def _prepare_cli(self): 26 | self.cli = TarSCM.Cli() 27 | if not os.path.exists(self.outdir): 28 | os.makedirs(self.outdir) 29 | self.cli.parse_args(['--outdir', self.outdir, '--scm', 'git']) 30 | self.cli.snapcraft = True 31 | 32 | def test_prep_tree_for_archive(self): 33 | tasks = TarSCM.Tasks(self.cli) 34 | scm_base = Scm(self.cli, tasks) 35 | basedir = os.path.join(self.tmp_dir, self.__class__.__name__) 36 | dir1 = os.path.join(basedir, "test1") 37 | scm_base.clone_dir = basedir 38 | os.makedirs(dir1) 39 | os.symlink('/', os.path.join(basedir, "test3")) 40 | 41 | with self.assertRaises(Exception) as ctx: 42 | scm_base.prep_tree_for_archive( 43 | "test2", 44 | basedir, 45 | "test1" 46 | ) 47 | 48 | if hasattr(ctx.exception, 'message'): 49 | msg = ctx.exception.message 50 | else: 51 | msg = ctx.exception 52 | 53 | six.assertRegex(self, str(msg), 'No such file or directory') 54 | 55 | scm_base.prep_tree_for_archive("test1", basedir, "test2") 56 | 57 | self.assertEqual(scm_base.arch_dir, os.path.join(basedir, "test2")) 58 | 59 | with self.assertRaises(SystemExit) as ctx: 60 | scm_base.prep_tree_for_archive("test3", basedir, "test2") 61 | 62 | def test_version_iso_cleanup(self): 63 | # Avoid get_repocache_hash failure in Scm.__init__ 64 | self.cli.url = "" 65 | scm_base = Scm(self.cli, None) 66 | self.assertEqual(scm_base.version_iso_cleanup("2.0.1-3", True), "2.0.1-3") 67 | self.assertEqual(scm_base.version_iso_cleanup("2.0.1-3", False), "2.0.13") 68 | self.assertEqual(scm_base.version_iso_cleanup("2.0.1-3"), "2.0.13") 69 | self.assertEqual(scm_base.version_iso_cleanup("1", True), "1") 70 | self.assertEqual(scm_base.version_iso_cleanup("1", False), "1") 71 | self.assertEqual(scm_base.version_iso_cleanup("1"), "1") 72 | self.assertEqual(scm_base.version_iso_cleanup("1.54-1.2", True), "1.54-1.2") 73 | self.assertEqual(scm_base.version_iso_cleanup("1.54-1.2", False), "1.541.2") 74 | self.assertEqual(scm_base.version_iso_cleanup("1.54-1.2"), "1.541.2") 75 | self.assertEqual(scm_base.version_iso_cleanup("2017-01-02 02:23:11 +0100", True), 76 | "20170102T022311") 77 | self.assertEqual(scm_base.version_iso_cleanup("2017-01-02 02:23:11 +0100", False), 78 | "20170102T022311") 79 | self.assertEqual(scm_base.version_iso_cleanup("2017-01-02 02:23:11 +0100"), 80 | "20170102T022311") 81 | -------------------------------------------------------------------------------- /tests/scmlogs.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from __future__ import print_function 3 | import glob 4 | import os 5 | import tempfile 6 | import io 7 | 8 | from utils import file_write_legacy 9 | 10 | 11 | class ScmInvocationLogs: 12 | """ 13 | Provides log files which tracks invocations of SCM binaries. The 14 | tracking is done via a wrapper around SCM to enable behaviour 15 | verification testing on tar_scm's repository caching code. This 16 | is cleaner than writing tests which look inside the cache, because 17 | then they become coupled to the cache's implementation, and 18 | require knowledge of where the cache lives etc. 19 | 20 | One instance should be constructed per unit test. If the test 21 | invokes the SCM binary multiple times, invoke next() in between 22 | each, so that a separate log file is used for each invocation - 23 | this allows more accurate fine-grained assertions on the 24 | invocation log. 25 | """ 26 | 27 | @classmethod 28 | def setup_bin_wrapper(cls, scm, tests_dir): 29 | wrapper_dir = tempfile.mkdtemp(dir="/tmp") 30 | 31 | wrapper_src = os.path.join(tests_dir, 'scm-wrapper') 32 | wrapper_dst = wrapper_dir + '/' + scm 33 | 34 | if not os.path.exists(wrapper_dst): 35 | os.symlink(wrapper_src, wrapper_dst) 36 | 37 | path = os.getenv('PATH') 38 | prepend = wrapper_dir + ':' 39 | 40 | if not path.startswith(prepend): 41 | new_path = prepend + path 42 | os.environ['PATH'] = new_path 43 | 44 | def __init__(self, scm, test_dir): 45 | self.scm = scm 46 | self.test_dir = test_dir 47 | self.counter = 0 48 | self.current_log_path = None 49 | 50 | self.unlink_existing_logs() 51 | 52 | def get_log_file_template(self): 53 | return '%s-invocation-%%s.log' % self.scm 54 | 55 | def get_log_path_template(self): 56 | return os.path.join(self.test_dir, self.get_log_file_template()) 57 | 58 | def unlink_existing_logs(self): 59 | pat = self.get_log_path_template() % '*' 60 | for log in glob.glob(pat): 61 | os.unlink(log) 62 | 63 | def get_log_file(self, identifier): 64 | if identifier: 65 | identifier = '-' + identifier 66 | return self.get_log_file_template() % \ 67 | ('%02d%s' % (self.counter, identifier)) 68 | 69 | def get_log_path(self, identifier): 70 | return os.path.join(self.test_dir, self.get_log_file(identifier)) 71 | 72 | def nextlog(self, identifier=''): 73 | self.counter += 1 74 | self.current_log_path = self.get_log_path(identifier) 75 | if os.path.exists(self.current_log_path): 76 | raise RuntimeError("%s already existed?!" % self.current_log_path) 77 | os.putenv('SCM_INVOCATION_LOG', self.current_log_path) 78 | os.environ['SCM_INVOCATION_LOG'] = self.current_log_path 79 | 80 | def annotate(self, msg): 81 | print(msg) 82 | file_write_legacy(self.current_log_path, '# ' + msg + "\n", 'a') 83 | 84 | def read(self): 85 | if not os.path.exists(self.current_log_path): 86 | return '' % self.scm 87 | with io.open(self.current_log_path, 'r', encoding="UTF-8") as log: 88 | loglines = log.readlines() 89 | return loglines 90 | -------------------------------------------------------------------------------- /tests/svnfixtures.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import os 4 | import stat 5 | from datetime import datetime 6 | 7 | from utils import mkfreshdir, quietrun, run_svn, file_write_legacy 8 | from fixtures import Fixtures 9 | 10 | 11 | class SvnFixtures(Fixtures): 12 | 13 | """Methods to create and populate a svn repository. 14 | 15 | svn tests use this class in order to have something to test against. 16 | """ 17 | 18 | SVN_COMMITTER_DATE = datetime.utcfromtimestamp( 19 | Fixtures.COMMITTER_DATE).isoformat() + ".000000Z" 20 | 21 | def init(self): 22 | self.wd_path = self.container_dir + '/wd' 23 | self.user_name = 'test' 24 | self.user_email = 'test@test.com' 25 | 26 | self.create_repo() 27 | self.checkout_repo() 28 | 29 | self.added = {} 30 | 31 | self.create_commits(2) 32 | 33 | def run(self, cmd): 34 | return run_svn(self.wd_path, cmd) 35 | 36 | def create_repo(self): 37 | quietrun('svnadmin create ' + self.repo_path) 38 | # allow revprop changes to explicitly set svn:date 39 | hook = self.repo_path + '/hooks/pre-revprop-change' 40 | file_write_legacy(hook, "#!/bin/sh\nexit 0;\n") 41 | 42 | sta = os.stat(hook) 43 | os.chmod(hook, sta.st_mode | stat.S_IEXEC) 44 | print("created repo %s" % self.repo_path) 45 | 46 | def checkout_repo(self): 47 | mkfreshdir(self.wd_path) 48 | quietrun('svn checkout %s %s' % (self.repo_url, self.wd_path)) 49 | self.wdir = self.wd_path 50 | 51 | def do_commit(self, wdir, new_rev, newly_created): # pylint: disable=W0612 52 | for new in newly_created: 53 | if new not in self.added: 54 | self.safe_run('add ' + new) 55 | self.added[new] = True 56 | self.safe_run('commit -m%d' % new_rev) 57 | self.safe_run('propset svn:date --revprop -r HEAD %s' % 58 | self.SVN_COMMITTER_DATE) 59 | return new_rev 60 | 61 | def get_metadata(self, formatstr): 62 | return self.safe_run("log -n1 %s" % formatstr)[0] 63 | 64 | def record_rev(self, *args): 65 | rev_num = args[0] 66 | self.revs[rev_num] = str(rev_num) 67 | self.scmlogs.annotate("Recorded rev %d" % rev_num) 68 | -------------------------------------------------------------------------------- /tests/svntests.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import os 4 | import re 5 | import io 6 | 7 | from gitsvntests import GitSvnTests 8 | from svnfixtures import SvnFixtures 9 | 10 | 11 | class SvnTests(GitSvnTests): 12 | 13 | """Unit tests for 'tar_scm --scm svn'. 14 | 15 | svn-specific tests are in this class. Other shared tests are 16 | included via the class inheritance hierarchy. 17 | """ 18 | 19 | scm = 'svn' 20 | initial_clone_command = 'svn (co|checkout) ' 21 | update_cache_command = 'svn up(date)?' 22 | sslverify_false_args = '--trust-server-cert' 23 | fixtures_class = SvnFixtures 24 | 25 | def default_version(self): 26 | return self.rev(2) 27 | 28 | def changesrevision(self, rev, abbrev=False): # noqa: E501 pylint: disable=W0613,R0201 29 | return rev 30 | 31 | def changesregex(self, rev): # pylint: disable=R0201 32 | return rev 33 | 34 | def tar_scm_args(self): # pylint: disable=R0201 35 | scm_args = [ 36 | '--changesgenerate', 'enable', 37 | '--versionformat', '0.6.%r', 38 | ] 39 | return scm_args 40 | 41 | def test_versionformat_rev(self): 42 | self.tar_scm_std('--versionformat', 'myrev%r.svn') 43 | self.assertTarOnly(self.basename(version='myrev2.svn')) 44 | 45 | def test_version_versionformat(self): 46 | self.tar_scm_std('--version', '3.0', '--versionformat', 'myrev%r.svn') 47 | self.assertTarOnly(self.basename(version='myrev2.svn')) 48 | 49 | def test_versionformat_revision(self): 50 | self.fixtures.create_commits(4) 51 | self.tar_scm_std('--versionformat', 'foo%r', '--revision', self.rev(2)) 52 | basename = self.basename(version='foo2') 53 | tar = self.assertTarOnly(basename) 54 | self.assertTarMemberContains(tar, basename + '/a', '2') 55 | 56 | def _check_servicedata(self, expected_dirents=2, revision=2): # noqa: E501 pylint: disable=W0613 57 | dirents = self.assertNumDirents(self.outdir, expected_dirents) 58 | self.assertTrue('_servicedata' in dirents, 59 | '_servicedata in %s' % repr(dirents)) 60 | infile = os.path.join(self.outdir, '_servicedata') 61 | with io.open(infile, 'r', encoding='UTF-8') as sdata: 62 | sdat = sdata.read() 63 | expected = ( 64 | r"" 65 | r"\s*" 66 | r"\s*%s" 67 | r"\s*([0-9].*)" 68 | r"\s*" 69 | r"\s*" % self.fixtures.repo_url 70 | ) 71 | reg = re.match(expected, sdat) 72 | if reg: 73 | print("matched") 74 | else: 75 | print("matched not") 76 | self.assertTrue(reg, "\n'%s'\n!~ /%s/" % (sdat, expected)) 77 | -------------------------------------------------------------------------------- /tests/tarfixtures.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from tests.fixtures import Fixtures 4 | 5 | 6 | class TarFixtures(Fixtures): 7 | 8 | """Methods to create and populate a tar directory. 9 | 10 | tar tests use this class in order to have something to test against. 11 | """ 12 | 13 | def init(self): 14 | pass 15 | 16 | def run(self, cmd): 17 | pass 18 | -------------------------------------------------------------------------------- /tests/tartests.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from __future__ import print_function 3 | 4 | import os 5 | import glob 6 | 7 | from utils import file_write_legacy 8 | 9 | from tests.tarfixtures import TarFixtures 10 | from tests.testenv import TestEnvironment 11 | from tests.testassertions import TestAssertions 12 | from tests.fake_classes import FakeCli, FakeTasks 13 | 14 | from TarSCM.scm.tar import Tar 15 | 16 | 17 | class TarTestCases(TestEnvironment, TestAssertions): 18 | """Unit tests for 'tar'. 19 | 20 | tar-specific tests are in this class. Other shared tests are 21 | included via the class inheritance hierarchy. 22 | """ 23 | 24 | scm = 'tar' 25 | fixtures_class = TarFixtures 26 | 27 | def test_tar_scm_finalize(self): 28 | wdir = self.pkgdir 29 | info = os.path.join(wdir, "test.obsinfo") 30 | print("INFOFILE: '%s'" % info) 31 | os.chdir(self.pkgdir) 32 | out_str = "name: pkgname\n" \ 33 | "version: 0.1.1\n" \ 34 | "mtime: 1476683264\n" \ 35 | "commit: fea6eb5f43841d57424843c591b6c8791367a9e5\n" 36 | file_write_legacy(info, out_str) 37 | 38 | src_dir = os.path.join(wdir, "pkgname") 39 | os.mkdir(src_dir) 40 | self.tar_scm_std() 41 | self.assertTrue(os.path.isdir(src_dir)) 42 | 43 | def test_tar_scm_no_finalize(self): # pylint: disable=no-self-use 44 | cli = FakeCli() 45 | tasks = FakeTasks() 46 | tar_obj = Tar(cli, tasks) 47 | tar_obj.finalize() 48 | 49 | 50 | def test_tar_scm_multiple_obsinfo(self): 51 | wdir = self.pkgdir 52 | info = os.path.join(wdir, "test1.obsinfo") 53 | print("INFOFILE: '%s'" % info) 54 | os.chdir(self.pkgdir) 55 | out_str = "name: pkgname1\n" \ 56 | "version: 0.1.1\n" \ 57 | "mtime: 1476683264\n" \ 58 | "commit: fea6eb5f43841d57424843c591b6c8791367a9e5\n" 59 | file_write_legacy(info, out_str) 60 | 61 | info = os.path.join(wdir, "test2.obsinfo") 62 | print("INFOFILE: '%s'" % info) 63 | os.chdir(self.pkgdir) 64 | out_str = "name: pkgname2\n" \ 65 | "version: 0.1.2\n" \ 66 | "mtime: 1476683264\n" \ 67 | "commit: fea6eb5f43841d57424843c591b6c8791367a9e5\n" 68 | file_write_legacy(info, out_str) 69 | 70 | src_dir = os.path.join(wdir, "pkgname1") 71 | os.mkdir(src_dir) 72 | src_dir = os.path.join(wdir, "pkgname2") 73 | os.mkdir(src_dir) 74 | self.tar_scm_std() 75 | self.assertTrue(os.path.isdir(src_dir)) 76 | os.chdir(self.outdir) 77 | files = glob.glob('*.tar') 78 | files.sort() 79 | expected = ['pkgname1-0.1.1-0.1.1.tar', 'pkgname2-0.1.2-0.1.2.tar'] 80 | self.assertEqual(files, expected) 81 | -------------------------------------------------------------------------------- /tests/tasks.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import os 4 | import inspect 5 | import shutil 6 | import unittest 7 | import io 8 | 9 | try: 10 | # pylint: disable=ungrouped-imports 11 | from unittest.mock import MagicMock 12 | except ImportError: 13 | from mock import MagicMock 14 | 15 | from tar_scm import TarSCM 16 | from tests.fake_classes import FakeSCM 17 | 18 | 19 | class TasksTestCases(unittest.TestCase): 20 | def setUp(self): 21 | self.basedir = os.path.abspath(os.path.dirname(__file__)) 22 | # os.getcwd() 23 | self.tests_dir = os.path.abspath(os.path.dirname(__file__)) 24 | self.tmp_dir = os.path.join(self.tests_dir, 'tmp') 25 | self.outdir = os.path.join(self.tmp_dir, 26 | self.__class__.__name__, 'out') 27 | self.cur_dir = None 28 | self._prepare_cli() 29 | 30 | def tearDown(self): 31 | if os.path.exists(self.outdir): 32 | shutil.rmtree(self.outdir) 33 | 34 | def _prepare_cli(self): 35 | self.cli = TarSCM.Cli() 36 | if not os.path.exists(self.outdir): 37 | os.makedirs(self.outdir) 38 | self.cli.parse_args(['--outdir', self.outdir, '--scm', 'git']) 39 | self.cli.snapcraft = True 40 | 41 | def _cd_fixtures_dir(self, *args): 42 | self.cur_dir = os.getcwd() 43 | cl_name = self.__class__.__name__ 44 | fn_name = inspect.stack()[1][3] 45 | if args: 46 | fn_name = args[0] 47 | 48 | try: 49 | os.chdir(os.path.join(self.basedir, 'fixtures', cl_name, fn_name)) 50 | except OSError: 51 | print("current working directory: %s" % os.getcwd()) 52 | raise 53 | 54 | def _restore_cwd(self): 55 | try: 56 | os.chdir(self.basedir) 57 | except OSError: 58 | print("failed to restore : current working directory: %s" % 59 | os.getcwd()) 60 | raise 61 | 62 | def _generate_tl_common(self, expected, func): 63 | self._cd_fixtures_dir(func) 64 | tasks = TarSCM.Tasks(self.cli) 65 | tasks.generate_list() 66 | self._restore_cwd() 67 | for key, val in expected.items(): 68 | self.assertEqual(tasks.task_list[0].__dict__[key], val) 69 | self.assertEqual(len(tasks.task_list), 1) 70 | 71 | def test_generate_tl_single_task(self): 72 | expected = { 73 | 'scm': 'bzr', 'clone_prefix': '_obs_', 'snapcraft': True, 74 | 'revision': None, 'url': 'lp:~mterry/libpipeline/printf', 75 | 'filename': 'libpipeline', 'use_obs_scm': True, 76 | 'outdir': self.cli.outdir, 'changesgenerate': False} 77 | self._generate_tl_common(expected, 'test_generate_tl_single_task') 78 | 79 | def test_generate_tl_st_appimage(self): 80 | '''Test generates task list with single task from appimage.yml''' 81 | self.cli.snapcraft = False 82 | self.cli.appimage = True 83 | expected = { 84 | 'scm': 'git', 'appimage': True, 85 | 'revision': None, 86 | 'url': 'https://github.com/probonopd/QtQuickApp.git', 87 | 'use_obs_scm': True, 88 | 'outdir': self.cli.outdir, 89 | 'changesgenerate': False 90 | } 91 | self._generate_tl_common(expected, 'test_generate_tl_st_appimage') 92 | 93 | def test_appimage_empty_build(self): 94 | self.cli.snapcraft = False 95 | self.cli.appimage = True 96 | self._cd_fixtures_dir() 97 | tasks = TarSCM.Tasks(self.cli) 98 | tasks.generate_list() 99 | 100 | def test_appimage_empty_build_git(self): 101 | self.cli.snapcraft = False 102 | self.cli.appimage = True 103 | self._cd_fixtures_dir() 104 | tasks = TarSCM.Tasks(self.cli) 105 | tasks.generate_list() 106 | 107 | def test_generate_tl_multi_tasks(self): 108 | expected = { 109 | 'libpipeline': { 110 | 'changesgenerate': False, 111 | 'clone_prefix': '_obs_', 112 | 'filename': 'libpipeline', 113 | 'outdir': self.cli.outdir, 114 | 'revision': None, 115 | 'scm': 'bzr', 116 | 'snapcraft': True, 117 | 'url': 'lp:~mterry/libpipeline/printf', 118 | 'use_obs_scm': True 119 | }, 120 | 'kanku': { 121 | 'changesgenerate': False, 122 | 'clone_prefix': '_obs_', 123 | 'filename': 'kanku', 124 | 'outdir': self.cli.outdir, 125 | 'revision': None, 126 | 'scm': 'git', 127 | 'snapcraft': True, 128 | 'url': 'git@github.com:M0ses/kanku', 129 | 'use_obs_scm': True 130 | } 131 | } 132 | self._cd_fixtures_dir() 133 | tasks = TarSCM.Tasks(self.cli) 134 | tasks.generate_list() 135 | # test values in the objects instead of objects 136 | for got in tasks.task_list: 137 | got_f = got.__dict__['filename'] 138 | for key in expected[got_f].keys(): 139 | self.assertEqual(got.__dict__[key], expected[got_f][key]) 140 | self._restore_cwd() 141 | 142 | def test_tasks_finalize(self): 143 | expected = '''\ 144 | apps: 145 | pipelinetest: 146 | command: ./bin/test 147 | description: 'This is an example package of an autotools project built with snapcraft 148 | 149 | using a remote source. 150 | 151 | ' 152 | name: pipelinetest 153 | parts: 154 | kanku: 155 | after: 156 | - libpipeline 157 | plugin: make 158 | source: kanku 159 | libpipeline: 160 | plugin: autotools 161 | source: libpipeline 162 | summary: Libpipeline example 163 | version: 1.0 164 | ''' # noqa 165 | self._cd_fixtures_dir() 166 | if not os.path.exists(self.cli.outdir): 167 | os.makedirs(self.cli.outdir) 168 | tasks = TarSCM.Tasks(self.cli) 169 | tasks.generate_list() 170 | tasks.finalize() 171 | self._restore_cwd() 172 | snf = os.path.join(self.cli.outdir,'_service:snapcraft:snapcraft.yaml') 173 | with io.open(snf, 'r', encoding='utf-8') as scf: 174 | got = scf.read() 175 | self.assertEqual(got, expected) 176 | 177 | def test_cleanup(self): 178 | tasks = TarSCM.Tasks(self.cli) 179 | cln = self.__class__.__name__ 180 | cn_dir = os.path.join(self.tmp_dir, cln) 181 | if not os.path.exists(self.tmp_dir): 182 | os.mkdir(self.tmp_dir) 183 | if not os.path.exists(cn_dir): 184 | os.mkdir(cn_dir) 185 | os.mkdir(os.path.join(cn_dir, 'test1')) 186 | tasks.cleanup_dirs.append(os.path.join(cn_dir, 'test1')) 187 | tasks.cleanup_dirs.append(os.path.join(cn_dir, 'does not exits')) 188 | tasks.cleanup_dirs.append(cn_dir) 189 | tasks.cleanup() 190 | self.assertEqual(os.path.exists(cn_dir), False) 191 | 192 | def test_get_version(self): 193 | scm = FakeSCM('0.0.1') 194 | tasks = TarSCM.Tasks(self.cli) 195 | tasks.scm_object = scm 196 | ver = tasks.get_version() 197 | self.assertEqual(ver, '0.0.1') 198 | self.cli.versionprefix = "r" 199 | ver = tasks.get_version() 200 | self.assertEqual(ver, 'r.0.0.1') 201 | 202 | def test_get_version_with_versionrw(self): 203 | '''Test for get_version with versionrewrite''' 204 | self.cli.versionrewrite_pattern = r'v(\d[\d\.]*)' 205 | self.cli.versionrewrite_replacement = '\\1-stable' 206 | scm = FakeSCM('v0.0.1') 207 | tasks = TarSCM.Tasks(self.cli) 208 | tasks.scm_object = scm 209 | ver = tasks.get_version() 210 | self.assertEqual(ver, '0.0.1-stable') 211 | 212 | def test_process_list(self): 213 | self.cli.snapcraft = False 214 | tasks = TarSCM.Tasks(self.cli) 215 | tasks.process_single_task = MagicMock(name='process_single_task') 216 | tasks.generate_list() 217 | tasks.process_list() 218 | -------------------------------------------------------------------------------- /tests/test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | ''' 4 | This CLI tool is responsible for running the tests. 5 | See TESTING.md for more information. 6 | ''' 7 | from __future__ import print_function 8 | 9 | import os 10 | import re 11 | import shutil 12 | import sys 13 | import unittest 14 | 15 | from tests.gittests import GitTests 16 | from tests.svntests import SvnTests 17 | from tests.hgtests import HgTests 18 | from tests.bzrtests import BzrTests 19 | from tests.testenv import TestEnvironment 20 | from tests.unittestcases import UnitTestCases 21 | from tests.tasks import TasksTestCases 22 | from tests.scm import SCMBaseTestCases 23 | from tests.tartests import TarTestCases 24 | from tests.archiveobscpiotestcases import ArchiveOBSCpioTestCases 25 | 26 | 27 | def str_to_class(string): 28 | '''Convert string into class''' 29 | return getattr(sys.modules[__name__], string) 30 | 31 | 32 | def prepare_testclasses(): 33 | tclasses = [ 34 | # If you are only interested in a particular VCS, you can 35 | # temporarily comment out any of these or use the env variable 36 | # TAR_SCM_TC= test.py 37 | # export TAR_SCM_TC=UnitTestCases,TasksTestCases,SCMBaseTestCases,GitTests,SvnTests,TarTestCases # noqa # pylint: disable=line-too-long 38 | HgTests, # disabled because of a lack of performance 39 | BzrTests, # disabled as bzr is no longer part of Factory 40 | UnitTestCases, 41 | TasksTestCases, 42 | ArchiveOBSCpioTestCases, 43 | SCMBaseTestCases, 44 | GitTests, 45 | SvnTests, 46 | TarTestCases 47 | ] 48 | 49 | # quite ugly to remove them here 50 | # but this way no linter complains about unused import 51 | tclasses.pop(1) 52 | tclasses.pop(1) 53 | 54 | if os.getenv('TAR_SCM_TC'): 55 | tclasses = [] 56 | for classname in os.environ['TAR_SCM_TC'].split(','): 57 | tclasses.append(str_to_class(classname)) 58 | 59 | return tclasses 60 | 61 | 62 | def prepare_testsuite(tclasses): 63 | testsuite = unittest.TestSuite() 64 | 65 | if len(sys.argv) == 1: 66 | for testclass in tclasses: 67 | all_tests = unittest.TestLoader().loadTestsFromTestCase(testclass) 68 | testsuite.addTests(all_tests) 69 | else: 70 | # By default this uses the CLI args as string or regexp 71 | # matches for names of git tests, but you can tweak this to run 72 | # specific tests, e.g.: 73 | # PYTHONPATH=.:tests tests/test.py test_versionformat 74 | for test_class in tclasses: 75 | to_run = {} 76 | for arg in sys.argv[1:]: 77 | rmatch = re.match('^/(.+)/$', arg) 78 | # pylint: disable=unnecessary-lambda-assignment 79 | if rmatch: 80 | # regexp mode 81 | regexp = rmatch.group(1) 82 | matcher = lambda t, r=regexp: re.search(r, t) 83 | else: 84 | matcher = lambda t, a=arg: t == a 85 | for tdir in dir(test_class): 86 | if not tdir.startswith('test_'): 87 | continue 88 | if matcher(tdir): 89 | to_run[tdir] = True 90 | 91 | for trun in to_run: 92 | testsuite.addTest(test_class(trun)) 93 | return testsuite 94 | 95 | 96 | def main(): 97 | test_classes = prepare_testclasses() 98 | suite = prepare_testsuite(test_classes) 99 | 100 | runner_args = { 101 | # 'verbosity': 2, 102 | # 'failfast': True, 103 | 'buffer': True 104 | } 105 | 106 | runner = unittest.TextTestRunner(**runner_args) 107 | result = runner.run(suite) 108 | 109 | # Cleanup: 110 | if result.wasSuccessful(): 111 | if os.path.exists(TestEnvironment.tmp_dir) and not os.environ.get('TAR_SCM_SKIP_CLEANUP'): 112 | shutil.rmtree(TestEnvironment.tmp_dir) 113 | sys.exit(0) 114 | else: 115 | print("Left temporary files in %s" % TestEnvironment.tmp_dir) 116 | print("You should remove these prior to the next test run.") 117 | sys.exit(1) 118 | 119 | 120 | if __name__ == '__main__': 121 | main() 122 | -------------------------------------------------------------------------------- /tests/testassertions.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # pylint: disable=C0103 3 | 4 | import os 5 | from pprint import pprint, pformat 6 | import re 7 | import sys 8 | import tarfile 9 | import unittest 10 | import six 11 | 12 | line_start = '(^|\n)' 13 | 14 | 15 | class TestAssertions(unittest.TestCase): 16 | 17 | """Library of test assertions used by tar_scm unit tests. 18 | 19 | This class augments Python's standard unittest.TestCase assertions 20 | with operations which the tar_scm unit tests commonly need. 21 | """ 22 | 23 | def assertNumDirents(self, wdir, expected, msg=''): 24 | dirents = os.listdir(wdir) 25 | got = len(dirents) 26 | if len(msg) > 0: 27 | msg += "\n" 28 | msg += 'expected %d file(s), got %d: %s' % \ 29 | (expected, got, pformat(dirents)) 30 | self.assertEqual(expected, got, msg) 31 | return dirents 32 | 33 | def assertNumTarEnts(self, tar, expected, msg=''): 34 | self.assertTrue(tarfile.is_tarfile(tar)) 35 | th = tarfile.open(tar) 36 | tarents = th.getmembers() 37 | got = len(tarents) 38 | if len(msg) > 0: 39 | msg += "\n" 40 | msg += 'expected %s to have %d entries, got %d:\n%s' % \ 41 | (tar, expected, got, pformat(tarents)) 42 | self.assertEqual(expected, got, msg) 43 | return th, tarents 44 | 45 | def assertDirentsMtime(self, entries): 46 | '''This test is disabled on Python 2.6 because tarfile is not able to 47 | directly change the mtime for an entry in the tarball.''' 48 | if sys.hexversion < 0x02070000: 49 | return 50 | for i in range(len(entries)): 51 | self.assertEqual(entries[i].mtime, 1234567890) 52 | 53 | def assertDirents(self, entries, top): 54 | self.assertEqual(entries[0].name, top) 55 | self.assertEqual(entries[1].name, top + '/a') 56 | self.assertEqual(entries[2].name, top + '/c') 57 | self.assertDirentsMtime(entries) 58 | 59 | def assertSubdirDirents(self, entries, top): 60 | self.assertEqual(entries[0].name, top) 61 | self.assertEqual(entries[1].name, top + '/b') 62 | self.assertDirentsMtime(entries) 63 | 64 | def assertStandardTar(self, tar, top): 65 | th, entries = self.assertNumTarEnts(tar, 5) 66 | entries.sort(key=lambda x: x.name) 67 | self.assertDirents(entries[:3], top) 68 | self.assertSubdirDirents(entries[3:], top + '/subdir') 69 | return th 70 | 71 | def assertSubdirTar(self, tar, top): 72 | th, entries = self.assertNumTarEnts(tar, 2) 73 | entries.sort(key=lambda x: x.name) 74 | self.assertSubdirDirents(entries, top) 75 | return th 76 | 77 | def assertIncludeSubdirTar(self, tar, top): 78 | th, entries = self.assertNumTarEnts(tar, 3) 79 | entries.sort(key=lambda x: x.name) 80 | self.assertEqual(entries[0].name, top) 81 | self.assertSubdirDirents(entries[1:], top + '/subdir') 82 | return th 83 | 84 | def checkTar(self, tar, tarbasename, toptardir=None, tarchecker=None): 85 | if not toptardir: 86 | toptardir = tarbasename 87 | if not tarchecker: 88 | tarchecker = self.assertStandardTar 89 | 90 | self.assertEqual(tar, '%s.tar' % tarbasename) 91 | tarpath = os.path.join(self.outdir, tar) 92 | return tarchecker(tarpath, toptardir) 93 | 94 | def assertTarOnly(self, tarbasename, **kwargs): 95 | dirents = self.assertNumDirents(self.outdir, 1) 96 | return self.checkTar(dirents[0], tarbasename, **kwargs) 97 | 98 | def assertTarAndDir(self, tarbasename, dirname=None, **kwargs): 99 | if not dirname: 100 | dirname = tarbasename 101 | 102 | dirents = self.assertNumDirents(self.outdir, 2) 103 | pprint(dirents) 104 | 105 | if dirents[0][-4:] == '.tar': 106 | tar = dirents[0] 107 | wdir = dirents[1] 108 | elif dirents[1][-4:] == '.tar': 109 | tar = dirents[1] 110 | wdir = dirents[0] 111 | else: 112 | self.fail('no .tar found in ' + self.outdir) 113 | 114 | self.assertEqual(wdir, dirname) 115 | self.assertTrue(os.path.isdir(os.path.join(self.outdir, wdir)), 116 | dirname + ' should be directory') 117 | 118 | return self.checkTar(tar, tarbasename, **kwargs) 119 | 120 | def assertTarMemberContains(self, tar, tarmember, contents): 121 | files = tar.extractfile(tarmember) 122 | self.assertEqual(contents, files.read().decode()) 123 | 124 | def assertRanInitialClone(self, logpath, loglines): 125 | self._find(logpath, loglines, 126 | self.initial_clone_command, self.update_cache_command) 127 | 128 | def assertSSLVerifyFalse(self, logpath, loglines): 129 | term = self.initial_clone_command + '.*' + self.sslverify_false_args 130 | self._find(logpath, loglines, 131 | term, 132 | self.sslverify_false_args + 'true') 133 | 134 | def assertRanUpdate(self, logpath, loglines): 135 | # exception for git - works different in cached mode 136 | should_not_find = self.initial_clone_command 137 | if self.__class__.__name__ == 'GitTests': 138 | should_not_find = None 139 | self._find(logpath, loglines, 140 | self.update_cache_command, should_not_find) 141 | 142 | def assertTarIsDeeply(self, tar, expected): 143 | if not os.path.isfile(tar): 144 | print("File '%s' not found" % tar) 145 | return False 146 | th = tarfile.open(tar) 147 | got = [] 148 | for mem in th.getmembers(): 149 | got.append(mem.name) 150 | result = got == expected 151 | if result == False: 152 | print("got: %r" % got) 153 | print("expected: %r" % expected) 154 | self.assertTrue(result) 155 | 156 | def _find(self, logpath, loglines, should_find, should_not_find): 157 | found = False 158 | regexp = re.compile('^' + should_find) 159 | for line in loglines: 160 | msg = \ 161 | "Shouldn't find /%s/ in %s; log was:\n" \ 162 | "----\n%s\n----\n" \ 163 | % (should_not_find, logpath, "".join(loglines)) 164 | if should_not_find: 165 | if six.PY2: 166 | self.assertNotRegexpMatches(line, should_not_find, msg) 167 | else: 168 | self.assertNotRegex(line, should_not_find, msg) 169 | if regexp.search(line): 170 | found = True 171 | msg = \ 172 | "Didn't find /%s/ in %s; log was:\n" \ 173 | "----\n%s\n----\n" \ 174 | % (regexp.pattern, logpath, "".join(loglines)) 175 | self.assertTrue(found, msg) 176 | -------------------------------------------------------------------------------- /tests/testenv.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # pylint: disable=C0103 3 | import os 4 | import shutil 5 | import sys 6 | import trace 7 | from utils import mkfreshdir 8 | from scmlogs import ScmInvocationLogs 9 | import TarSCM 10 | 11 | try: 12 | from StringIO import StringIO 13 | except: 14 | from io import StringIO 15 | 16 | 17 | class TestEnvironment: 18 | """Framework for testing tar_scm. 19 | 20 | This class provides methods for: 21 | 22 | - setting up and tearing down a test environment similar to what 23 | 'osc service' would provide, and 24 | - running tar_scm inside that environment. 25 | """ 26 | tests_dir = os.path.abspath(os.path.dirname(__file__)) # os.getcwd() 27 | tmp_dir = os.path.join(tests_dir, 'tmp') 28 | is_setup = False 29 | 30 | @classmethod 31 | def tar_scm_bin(cls): 32 | tar_scm = os.path.join(cls.tests_dir, '..', 'tar_scm.py') 33 | if not os.path.isfile(tar_scm): 34 | msg = "Failed to find tar_scm executable at " + tar_scm 35 | raise RuntimeError(msg) 36 | return tar_scm 37 | 38 | @classmethod 39 | def setupClass(cls): 40 | # deliberately not setUpClass - we emulate the behaviour 41 | # to support Python < 2.7 42 | if cls.is_setup: 43 | return 44 | print("--v-v-- begin setupClass for %s --v-v--" % cls.__name__) 45 | ScmInvocationLogs.setup_bin_wrapper(cls.scm, cls.tests_dir) 46 | os.putenv('TAR_SCM_CLEAN_ENV', 'yes') 47 | os.environ['TAR_SCM_CLEAN_ENV'] = 'yes' 48 | cls.is_setup = True 49 | print("--^-^-- end setupClass for %s --^-^--" % cls.__name__) 50 | print() 51 | 52 | def calcPaths(self): 53 | if not self._testMethodName.startswith('test_'): 54 | msg = "unexpected test method name: " + self._testMethodName 55 | raise RuntimeError(msg) 56 | 57 | self.test_dir = os.path.join(self.tmp_dir, self.scm, self.test_name) 58 | self.pkgdir = os.path.join(self.test_dir, 'pkg') 59 | self.homedir = os.path.join(self.test_dir, 'home') 60 | self.outdir = os.path.join(self.test_dir, 'out') 61 | self.cachedir = os.path.join(self.test_dir, 'cache') 62 | 63 | def setUp(self): 64 | print() 65 | print("=" * 70) 66 | print(self._testMethodName) 67 | print("=" * 70) 68 | print() 69 | 70 | self.test_name = self._testMethodName[5:] 71 | 72 | self.setupClass() 73 | 74 | print("--v-v-- begin setUp for %s --v-v--" % self.test_name) 75 | 76 | self.calcPaths() 77 | 78 | self.scmlogs = ScmInvocationLogs(self.scm, self.test_dir) 79 | self.scmlogs.nextlog('fixtures') 80 | 81 | self.initDirs() 82 | 83 | self.fixtures = self.fixtures_class(self.test_dir, self.scmlogs) 84 | self.fixtures.setup() 85 | 86 | self.scmlogs.nextlog('start-test') 87 | self.scmlogs.annotate('Starting %s test' % self.test_name) 88 | 89 | os.putenv('CACHEDIRECTORY', self.cachedir) 90 | os.environ['CACHEDIRECTORY'] = self.cachedir 91 | print("--^-^-- end setUp for %s --^-^--" % self.test_name) 92 | 93 | def initDirs(self): 94 | # pkgdir persists between tests to simulate real world use 95 | # (although a test can choose to invoke mkfreshdir) 96 | persistent_dirs = [self.pkgdir, self.homedir] 97 | for i_dir in persistent_dirs: 98 | if not os.path.exists(i_dir): 99 | os.makedirs(i_dir) 100 | 101 | # Tests should not depend on the contents of $HOME 102 | os.putenv('HOME', self.homedir) 103 | os.environ['HOME'] = self.homedir 104 | 105 | for subdir in ('repo', 'repourl', 'incoming'): 106 | mkfreshdir(os.path.join(self.cachedir, subdir)) 107 | 108 | def disableCache(self): 109 | os.unsetenv('CACHEDIRECTORY') 110 | os.environ['CACHEDIRECTORY'] = "" 111 | 112 | def tar_scm_std(self, *args, **kwargs): 113 | return self.tar_scm(self.stdargs(*args), **kwargs) 114 | 115 | def tar_scm_std_fail(self, *args): 116 | return self.tar_scm(self.stdargs(*args), should_succeed=False) 117 | 118 | def stdargs(self, *args): 119 | return [ 120 | '--url', self.fixtures.repo_url, 121 | '--scm', self.scm 122 | ] + list(args) 123 | 124 | def tar_scm(self, args, should_succeed=True): 125 | # simulate new temporary outdir for each tar_scm invocation 126 | mkfreshdir(self.outdir) 127 | 128 | # osc launches source services with cwd as pkg dir 129 | # (see run_source_services() in osc/core.py) 130 | print("chdir to pkgdir: %s" % self.pkgdir) 131 | os.chdir(self.pkgdir) 132 | 133 | cmdargs = args + ['--outdir', self.outdir] 134 | sys.argv = [self.tar_scm_bin()] + cmdargs 135 | 136 | old_stdout = sys.stdout 137 | mystdout = StringIO() 138 | sys.stdout = mystdout 139 | 140 | old_stderr = sys.stderr 141 | mystderr = StringIO() 142 | sys.stderr = mystderr 143 | 144 | cmdstr = " ".join(sys.argv) 145 | print() 146 | print(">>>>>>>>>>>") 147 | print("Running %s" % cmdstr) 148 | print() 149 | print("start TarSCM.run") 150 | succeeded = True 151 | ret = 0 152 | try: 153 | TarSCM.run() 154 | except SystemExit as exp: 155 | print("raised system exit %r" % exp) 156 | if exp.code == 0: 157 | print("exp.code is ok") 158 | ret = 0 159 | succeeded = True 160 | else: 161 | print("exp.code is not 0") 162 | sys.stderr.write(exp.code) 163 | ret = 1 164 | succeeded = False 165 | except (NameError, AttributeError) as exp: 166 | sys.stderr.write(exp) 167 | ret = 1 168 | succeeded = False 169 | except Exception as exp: 170 | print("Raised Exception %r" % exp) 171 | if hasattr(exp, 'message'): 172 | msg = exp.message 173 | else: 174 | msg = exp 175 | sys.stderr.write(str(msg)) 176 | ret = 1 177 | succeeded = False 178 | finally: 179 | sys.stdout = old_stdout 180 | sys.stderr = old_stderr 181 | 182 | stdout = mystdout.getvalue() 183 | stderr = mystderr.getvalue() 184 | 185 | if stdout: 186 | print("--v-v-- begin STDOUT from tar_scm --v-v--") 187 | print(stdout) 188 | print("--^-^-- end STDOUT from tar_scm --^-^--") 189 | 190 | if stderr: 191 | print("\n") 192 | print("--v-v-- begin STDERR from tar_scm --v-v--") 193 | print(stderr) 194 | print("--^-^-- end STDERR from tar_scm --^-^--") 195 | print("succeeded: %r - should_succeed %r" % 196 | (succeeded, should_succeed)) 197 | result = ("succeed" if should_succeed else "fail") 198 | self.assertEqual(succeeded, should_succeed, 199 | "expected tar_scm to " + result) 200 | 201 | return (stdout, stderr, ret) 202 | 203 | def rev(self, rev): 204 | return self.fixtures.revs[rev] 205 | 206 | def timestamps(self, rev): 207 | return self.fixtures.timestamps[rev] 208 | -------------------------------------------------------------------------------- /tests/utils.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Simple utility functions to help executing processes. 4 | from __future__ import print_function 5 | 6 | import os 7 | import re 8 | import io 9 | import shutil 10 | import subprocess 11 | import sys 12 | import six 13 | 14 | 15 | def mkfreshdir(path): 16 | if not re.search('.{10}/tmp(/|$)', path): 17 | raise RuntimeError('unsafe call: mkfreshdir(%s)' % path) 18 | 19 | cwd = os.getcwd() 20 | os.chdir('/') 21 | if os.path.exists(path): 22 | shutil.rmtree(path) 23 | os.makedirs(path) 24 | os.chdir(cwd) 25 | 26 | def check_locale(loc): 27 | try: 28 | aloc_tmp = subprocess.check_output(['locale', '-a']) 29 | except AttributeError: 30 | aloc_tmp, _ = subprocess.Popen(['locale', '-a'], 31 | stdout=subprocess.PIPE, 32 | stderr=subprocess.STDOUT).communicate() 33 | aloc = {} 34 | 35 | for tloc in aloc_tmp.split(b'\n'): 36 | aloc[tloc] = 1 37 | 38 | for tloc in loc: 39 | print("Checking .... %s"%tloc, file=sys.stderr) 40 | try: 41 | if aloc[tloc.encode()]: 42 | return tloc 43 | except KeyError: 44 | pass 45 | 46 | return 'C' 47 | 48 | def run_cmd(cmd): 49 | use_locale = check_locale(["en_US.utf8", 'C.utf8']) 50 | os.environ['LANG'] = use_locale 51 | os.environ['LC_ALL'] = use_locale 52 | if six.PY3: 53 | cmd = cmd.encode('UTF-8') 54 | proc = subprocess.Popen( 55 | cmd, 56 | shell=True, 57 | stdout=subprocess.PIPE, 58 | stderr=subprocess.PIPE) 59 | 60 | (stdout, stderr) = proc.communicate() 61 | return (stdout, stderr, proc.returncode) 62 | 63 | 64 | def quietrun(cmd): 65 | (stdout, stderr, ret) = run_cmd(cmd) 66 | if ret != 0: 67 | print(cmd, " failed!") 68 | print(stdout) 69 | print(stderr) 70 | return (stdout, stderr, ret) 71 | 72 | 73 | def run_scm(scm, repo, args): 74 | cmd = '%s %s' % (scm, args) 75 | if repo: 76 | cmd = 'cd %s && %s' % (repo, cmd) 77 | # return subprocess.check_output(cmd, shell=True) 78 | return quietrun(cmd) 79 | 80 | 81 | def run_git(args): 82 | return run_scm('git', None, args) 83 | 84 | 85 | def run_svn(repo, args): 86 | return run_scm('svn', repo, args) 87 | 88 | 89 | def run_hg(repo, args): 90 | return run_scm('hg', repo, args) 91 | 92 | 93 | def run_bzr(repo, args): 94 | return run_scm('bzr', repo, args) 95 | 96 | def file_write_legacy(fname, string, *args): 97 | '''function to write string to file python 2/3 compatible''' 98 | mode = 'w' 99 | if args: 100 | mode = args[0] 101 | 102 | with io.open(fname, mode, encoding='utf-8') as outfile: 103 | # 'str().encode().decode()' is required for pyhton 2/3 compatibility 104 | outfile.write(str(string).encode('UTF-8').decode('UTF-8')) 105 | --------------------------------------------------------------------------------