├── .dockerignore ├── .github └── workflows │ ├── cicd.yml │ ├── cleanup.yml │ ├── contributors.yml │ └── release.yml ├── .gitignore ├── CHANGELOG.md ├── CODEOWNERS ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── VERSION ├── docs ├── Makefile ├── _static │ └── .gitkeep ├── client.md ├── conf.py ├── configuration │ ├── appearance.md │ ├── application_settings.md │ ├── features.md │ ├── index.md │ ├── instance_variables.md │ └── license.md ├── faq.md ├── index.rst ├── install.md ├── make.bat └── usage.md ├── examples ├── basic │ └── gitlab.yml ├── environment_variables │ ├── external_auth.yml │ └── gitlab.yml ├── gitlab.cfg └── modularized │ ├── appearance.yml │ ├── gitlab.yml │ ├── license.yml │ └── settings │ ├── asset_proxy.yml │ ├── external_auth.yml │ ├── settings.yml │ └── tos.md ├── gcasc ├── __init__.py ├── appearance.py ├── base.py ├── bin │ └── gcasc ├── config.py ├── exceptions.py ├── features.py ├── instance_variables.py ├── license.py ├── settings.py └── utils │ ├── __init__.py │ ├── diff.py │ ├── logger.py │ ├── objects.py │ ├── os.py │ ├── strings.py │ ├── validators.py │ ├── yaml_env.py │ └── yaml_include.py ├── renovate.json ├── requirements.txt ├── rtd-requirements.txt ├── setup.py ├── test-requirements.txt ├── tests ├── __init__.py ├── appearance_test.py ├── config_test.py ├── data │ ├── appearance_valid.yml │ ├── dummycert.crt │ ├── dummykey.key │ ├── features_invalid.yml │ ├── features_valid.yml │ ├── features_valid_canaries.yml │ ├── features_valid_canary.yml │ ├── gitlab.yml │ ├── gitlab_config_invalid.cfg │ ├── gitlab_config_valid.cfg │ ├── instance_variables_invalid.yml │ ├── instance_variables_valid.yml │ ├── license_invalid_1.yml │ ├── license_valid.yml │ ├── settings_valid.yml │ ├── yaml_env.yml │ ├── yaml_include.yml │ ├── yaml_include_f1.yml │ ├── yaml_include_f2.yml │ └── yaml_include_txt.md ├── diff_test.py ├── features_test.py ├── gcasc_test.py ├── helpers.py ├── instance_variables_test.py ├── license_test.py ├── settings_test.py ├── yaml_env_test.py └── yaml_include_test.py └── tox.ini /.dockerignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | MANIFEST 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | .pytest_cache/ 49 | out_report.xml 50 | 51 | # Translations 52 | *.mo 53 | *.pot 54 | 55 | # Django stuff: 56 | *.log 57 | local_settings.py 58 | db.sqlite3 59 | 60 | # Flask stuff: 61 | instance/ 62 | .webassets-cache 63 | 64 | # Scrapy stuff: 65 | .scrapy 66 | 67 | # Sphinx documentation 68 | docs/_build/ 69 | 70 | # PyBuilder 71 | target/ 72 | 73 | # Jupyter Notebook 74 | .ipynb_checkpoints 75 | 76 | # pyenv 77 | .python-version 78 | 79 | # celery beat schedule file 80 | celerybeat-schedule 81 | 82 | # SageMath parsed files 83 | *.sage.py 84 | 85 | # Environments 86 | .env 87 | .venv 88 | env/ 89 | venv/ 90 | ENV/ 91 | env.bak/ 92 | venv.bak/ 93 | 94 | # Spyder project settings 95 | .spyderproject 96 | .spyproject 97 | 98 | # Rope project settings 99 | .ropeproject 100 | 101 | # mkdocs documentation 102 | /site 103 | 104 | # mypy 105 | .mypy_cache/ 106 | 107 | include 108 | *python* 109 | 110 | .idea 111 | 112 | # secret 113 | *secret* 114 | *key 115 | id_rsa -------------------------------------------------------------------------------- /.github/workflows/cicd.yml: -------------------------------------------------------------------------------- 1 | name: CI/CD 2 | on: 3 | push: 4 | branches: 5 | - master 6 | tags: 7 | - '*' 8 | paths-ignore: 9 | - 'docs/**' 10 | - 'examples/**' 11 | pull_request: 12 | branches: 13 | - '*' 14 | 15 | jobs: 16 | cleanup-runs: 17 | name: Cleanup previous runs 18 | runs-on: ubuntu-latest 19 | steps: 20 | - uses: rokroskar/workflow-run-cleanup-action@master 21 | env: 22 | GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" 23 | 24 | lint: 25 | name: Lint 26 | runs-on: ubuntu-latest 27 | steps: 28 | - uses: actions/checkout@v2 29 | - name: Set up Python 3.7 30 | uses: actions/setup-python@v2 31 | with: 32 | python-version: 3.7 33 | - name: Install Tox 34 | run: pip install tox 35 | - name: Run linter 36 | run: make lint 37 | 38 | test: 39 | name: Test 40 | runs-on: ubuntu-latest 41 | strategy: 42 | matrix: 43 | python-version: [3.6, 3.7, 3.8] 44 | 45 | steps: 46 | - uses: actions/checkout@v2 47 | - name: Set up Python ${{ matrix.python-version }} 48 | uses: actions/setup-python@v2 49 | with: 50 | python-version: ${{ matrix.python-version }} 51 | - name: Install Tox 52 | run: pip install tox 53 | - name: Run tests 54 | run: tox -e py 55 | - name: Publish Test Report 56 | uses: mikepenz/action-junit-report@v1 57 | if: ${{ github.event.pull_request && matrix.python-version == '3.8' }} 58 | with: 59 | report_paths: './out_report.xml' 60 | github_token: ${{ secrets.GITHUB_TOKEN }} 61 | - name: Report coverage 62 | uses: codecov/codecov-action@v1 63 | env: 64 | PYTHON_VERSION: ${{ matrix.python-version }} 65 | with: 66 | file: ./coverage.xml 67 | flags: unittest 68 | env_vars: PYTHON_VERSION 69 | fail_ci_if_error: false 70 | 71 | build: 72 | name: Build and publish to PyPi 73 | runs-on: ubuntu-latest 74 | needs: [lint, test] 75 | steps: 76 | - uses: actions/checkout@v2 77 | - name: Set up Python 3.8 78 | uses: actions/setup-python@v2 79 | with: 80 | python-version: 3.8 81 | - name: Build Python package 82 | run: make build 83 | - name: Publish package 84 | if: contains(github.ref, 'refs/tags/') 85 | env: 86 | TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} 87 | TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} 88 | run: make publish 89 | 90 | docker: 91 | name: Build and push Docker image 92 | runs-on: ubuntu-latest 93 | needs: [lint, test] 94 | steps: 95 | - uses: actions/checkout@v2 96 | - name: Set up QEMU 97 | uses: docker/setup-qemu-action@v1 98 | - name: Set up Docker Buildx 99 | uses: docker/setup-buildx-action@v1 100 | - name: Login to DockerHub 101 | uses: docker/login-action@v1 102 | if: ${{ github.ref == 'refs/heads/master' || contains(github.ref, 'refs/tags/') }} 103 | with: 104 | username: ${{ secrets.DOCKERHUB_USERNAME }} 105 | password: ${{ secrets.DOCKERHUB_TOKEN }} 106 | - name: Extract Docker metadata 107 | id: docker_meta 108 | uses: crazy-max/ghaction-docker-meta@v1 109 | with: 110 | images: hoffmannlaroche/gcasc 111 | tag-semver: | 112 | {{major}} 113 | {{major}}.{{minor}} 114 | {{major}}.{{minor}}.{{patch}} 115 | - name: Build and push 116 | id: docker_build 117 | uses: docker/build-push-action@v2 118 | with: 119 | push: ${{ github.ref == 'refs/heads/master' || contains(github.ref, 'refs/tags/') }} 120 | tags: ${{ steps.docker_meta.outputs.tags }} 121 | labels: ${{ steps.docker_meta.outputs.labels }} 122 | -------------------------------------------------------------------------------- /.github/workflows/cleanup.yml: -------------------------------------------------------------------------------- 1 | name: Clean repository 2 | on: 3 | schedule: 4 | - cron: "0 4 * * *" 5 | 6 | jobs: 7 | 8 | cleanup: 9 | name: Cleanup repository from stale draft, PRs and issues 10 | runs-on: ubuntu-latest 11 | 12 | steps: 13 | - name: Delete draft releases 14 | uses: hugo19941994/delete-draft-releases@v0.1.0 15 | env: 16 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 17 | 18 | - name: Cleanup stale issues and PRs 19 | uses: actions/stale@v3 20 | with: 21 | repo-token: ${{ secrets.GITHUB_TOKEN }} 22 | stale-issue-message: 'This issue is stale because it has been open 30 days with no activity. Remove stale label or comment or this will be closed in 5 days' 23 | stale-pr-message: 'This PR is stale because it has been open 30 days with no activity. Remove stale label or comment or this will be closed in 5 days' 24 | days-before-stale: 30 25 | days-before-close: 5 26 | stale-issue-label: 'no-activity' 27 | exempt-issue-labels: 'work-in-progress' 28 | stale-pr-label: 'no-activity' 29 | exempt-pr-labels: 'work-in-progress' -------------------------------------------------------------------------------- /.github/workflows/contributors.yml: -------------------------------------------------------------------------------- 1 | name: Contributors 2 | on: 3 | push: 4 | branches: 5 | - master 6 | 7 | jobs: 8 | contributors: 9 | name: Add Conributors 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@v2 13 | - name: Contribute List 14 | uses: akhilmhdh/contributors-readme-action@v2.0.2 15 | env: 16 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 17 | with: 18 | commit_message: 'docs(readme): update contributors' 19 | committer_username: 'Contribution Bot' 20 | image_size: 80 21 | columns_per_row: 7 -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Release 2 | on: 3 | push: 4 | branches: 5 | - master 6 | paths: 7 | - 'VERSION' 8 | 9 | jobs: 10 | release: 11 | name: Release 12 | runs-on: ubuntu-latest 13 | steps: 14 | - uses: actions/checkout@v2 15 | - name: Read package.json 16 | id: version 17 | uses: juliangruber/read-file-action@v1 18 | with: 19 | path: ./VERSION 20 | - name: New version 21 | run: echo ${{ steps.version.outputs.content }} 22 | - name: Update changelog 23 | uses: thomaseizinger/keep-a-changelog-new-release@v1 24 | with: 25 | version: ${{ steps.version.outputs.content }} 26 | - name: Commit changes 27 | uses: EndBug/add-and-commit@v5 28 | with: 29 | author_name: Changelog Bot 30 | author_email: changelog-bot@github.com 31 | message: "docs(changelog): update changelog for version ${{ steps.version.outputs.content }}" 32 | add: "CHANGELOG.md" 33 | env: 34 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 35 | - name: Get Changelog Entry 36 | id: changelog_reader 37 | uses: mindsers/changelog-reader-action@v2 38 | with: 39 | validation_depth: 0 40 | version: ${{ steps.version.outputs.content }} 41 | path: ./CHANGELOG.md 42 | - name: Create Release 43 | id: create_release 44 | uses: actions/create-release@v1 45 | env: 46 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 47 | with: 48 | tag_name: ${{ steps.version.outputs.content }} 49 | release_name: v${{ steps.version.outputs.content }} 50 | body: ${{ steps.changelog_reader.outputs.changes }} 51 | prerelease: ${{ steps.changelog_reader.outputs.status == 'prereleased' }} 52 | draft: ${{ steps.changelog_reader.outputs.status == 'unreleased' }} -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | MANIFEST 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | .pytest_cache/ 49 | out_report.xml 50 | 51 | # Translations 52 | *.mo 53 | *.pot 54 | 55 | # Django stuff: 56 | *.log 57 | local_settings.py 58 | db.sqlite3 59 | 60 | # Flask stuff: 61 | instance/ 62 | .webassets-cache 63 | 64 | # Scrapy stuff: 65 | .scrapy 66 | 67 | # Sphinx documentation 68 | docs/_build/ 69 | 70 | # PyBuilder 71 | target/ 72 | 73 | # Jupyter Notebook 74 | .ipynb_checkpoints 75 | 76 | # pyenv 77 | .python-version 78 | 79 | # celery beat schedule file 80 | celerybeat-schedule 81 | 82 | # SageMath parsed files 83 | *.sage.py 84 | 85 | # Environments 86 | .env 87 | .venv 88 | env/ 89 | venv/ 90 | ENV/ 91 | env.bak/ 92 | venv.bak/ 93 | 94 | # Spyder project settings 95 | .spyderproject 96 | .spyproject 97 | 98 | # Rope project settings 99 | .ropeproject 100 | 101 | # mkdocs documentation 102 | /site 103 | 104 | # mypy 105 | .mypy_cache/ 106 | 107 | include 108 | *python* 109 | 110 | .idea 111 | *.iml 112 | 113 | # sphinx documentation 114 | docs/_build 115 | 116 | # secret 117 | *secret* 118 | *key 119 | id_rsa -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | This document contains comprehensive information of the new features, enhancements, 4 | fixes and other changes of GitLab Configuration as Code. 5 | 6 | ## [Unreleased] 7 | 8 | ### Fixed 9 | 10 | - Adjust CI variable validation regex to the one GitLab uses 11 | 12 | ## [0.6.2] - 2021-04-09 13 | 14 | ## [0.6.1] - 2021-01-22 15 | 16 | ### Fixed 17 | 18 | - Add hyphen '-' support to Instance CI/CD Variables 19 | 20 | ## [0.6.0] - 2020-11-26 21 | 22 | ### Added 23 | 24 | - Support for configuring Instance CI/CD Variables 25 | 26 | ### Changed 27 | 28 | - Validation of resources using JSON Schema 29 | - Switch CI from Travis to GitHub Actions 30 | 31 | ### Security 32 | 33 | - Do not log values of any variables, because this may lead to leak of secrets 34 | 35 | ## [0.5.0] - 2020-04-14 36 | 37 | ### Added 38 | 39 | - Use `!include` directive with path relative to GitLab configuration file path 40 | 41 | ## [0.4.0] - 2020-03-12 42 | 43 | ### Added 44 | 45 | - Support for configuring Feature Flags 46 | - Support for mixing GitLab client configuration in file and environment variables 47 | 48 | ## [0.3.1] - 2020-02-06 49 | 50 | ### Fixed 51 | 52 | - Calculation of key prefixes in `UpdateOnlyConfigurer` 53 | 54 | ## [0.3.0] - 2020-02-04 55 | 56 | ### Added 57 | 58 | - Support for configuring Appearance 59 | 60 | ### Changed 61 | 62 | - Updated dependency on `python-gitlab` 63 | - Code modularization 64 | 65 | ## [0.2.0] - 2019-11-28 66 | 67 | ### Added 68 | 69 | - Documentation available under 70 | 71 | ## [0.1.0] - 2019-11-28 72 | 73 | ### Added 74 | 75 | - Initial release with support for application settings and license 76 | 77 | [Unreleased]: https://github.com/Roche/gitlab-configuration-as-code/compare/0.6.2...HEAD 78 | 79 | [0.6.2]: https://github.com/Roche/gitlab-configuration-as-code/compare/0.6.1...0.6.2 80 | 81 | [0.6.1]: https://github.com/Roche/gitlab-configuration-as-code/compare/0.6.0...0.6.1 82 | 83 | [0.6.0]: https://github.com/Roche/gitlab-configuration-as-code/compare/0.5.0...0.6.0 84 | -------------------------------------------------------------------------------- /CODEOWNERS: -------------------------------------------------------------------------------- 1 | * mateusz.filipowicz@roche.com 2 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as 6 | contributors and maintainers pledge to making participation in our project and 7 | our community a harassment-free experience for everyone, regardless of age, body 8 | size, disability, ethnicity, sex characteristics, gender identity and expression, 9 | level of experience, education, socio-economic status, nationality, personal 10 | appearance, race, religion, or sexual identity and orientation. 11 | 12 | ## Our Standards 13 | 14 | Examples of behavior that contributes to creating a positive environment 15 | include: 16 | 17 | * Using welcoming and inclusive language 18 | * Being respectful of differing viewpoints and experiences 19 | * Gracefully accepting constructive criticism 20 | * Focusing on what is best for the community 21 | * Showing empathy towards other community members 22 | 23 | Examples of unacceptable behavior by participants include: 24 | 25 | * The use of sexualized language or imagery and unwelcome sexual attention or 26 | advances 27 | * Trolling, insulting/derogatory comments, and personal or political attacks 28 | * Public or private harassment 29 | * Publishing others' private information, such as a physical or electronic 30 | address, without explicit permission 31 | * Other conduct which could reasonably be considered inappropriate in a 32 | professional setting 33 | 34 | ## Our Responsibilities 35 | 36 | Project maintainers are responsible for clarifying the standards of acceptable 37 | behavior and are expected to take appropriate and fair corrective action in 38 | response to any instances of unacceptable behavior. 39 | 40 | Project maintainers have the right and responsibility to remove, edit, or 41 | reject comments, commits, code, wiki edits, issues, and other contributions 42 | that are not aligned to this Code of Conduct, or to ban temporarily or 43 | permanently any contributor for other behaviors that they deem inappropriate, 44 | threatening, offensive, or harmful. 45 | 46 | ## Scope 47 | 48 | This Code of Conduct applies both within project spaces and in public spaces 49 | when an individual is representing the project or its community. Examples of 50 | representing a project or community include using an official project e-mail 51 | address, posting via an official social media account, or acting as an appointed 52 | representative at an online or offline event. Representation of a project may be 53 | further defined and clarified by project maintainers. 54 | 55 | ## Enforcement 56 | 57 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 58 | reported by contacting the project team at `global.open_source@roche.com`. All 59 | complaints will be reviewed and investigated and will result in a response that 60 | is deemed necessary and appropriate to the circumstances. The project team is 61 | obligated to maintain confidentiality with regard to the reporter of an incident. 62 | Further details of specific enforcement policies may be posted separately. 63 | 64 | Project maintainers who do not follow or enforce the Code of Conduct in good 65 | faith may face temporary or permanent repercussions as determined by other 66 | members of the project's leadership. 67 | 68 | ## Attribution 69 | 70 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, 71 | available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html 72 | 73 | [homepage]: https://www.contributor-covenant.org 74 | 75 | For answers to common questions about this code of conduct, see 76 | https://www.contributor-covenant.org/faq 77 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | When contributing to this repository, please first discuss the change you wish to make via issue, 4 | email, or any other method with the owners of this repository before making a change. Or at 5 | 6 | Please note we have a code of conduct, please follow it in all your interactions with the project. 7 | 8 | - [Feature Requests](#feature) 9 | - [Issues and Bugs](#issue) 10 | - [Submission Guidelines](#submit) 11 | - [Coding Rules](#rules) 12 | - [Git Commit Guidelines](#commit) 13 | 14 | ## Feature Requests 15 | You can request a new feature by submitting a ticket to our [Github issues](https://github.com/Roche/gitlab-configuration-as-code/issues/new). 16 | If you would like to implement a new feature then open up a ticket, explain your change in the description 17 | and you can propose a Pull Request straight away. 18 | 19 | Before raising a new feature requests, you can [browse existing requests](https://github.com/Roche/gitlab-configuration-as-code/issues) 20 | to save us time removing duplicates. 21 | 22 | ## Issues and Bugs 23 | If you find a bug in the source code or a mistake in the documentation, you can help us by [ 24 | submitting a ticket](https://github.com/Roche/gitlab-configuration-as-code/issues/new). 25 | **Even better**, if you could submit a Pull Request to our repo fixing the issue. 26 | 27 | **Please see the Submission Guidelines below**. 28 | 29 | ## Submission Guidelines 30 | 31 | ### [Submitting an Issue](https://opensource.guide/how-to-contribute/#opening-an-issue) 32 | Before you submit your issue search the [backlog](https://github.com/Roche/gitlab-configuration-as-code/issues), 33 | maybe your question was already answered or is already there in backlog. 34 | 35 | Providing the following information will increase the chances of your issue being dealt with quickly: 36 | 37 | * **Overview of the issue** - if an error is being thrown a stack trace helps 38 | * **Motivation for or Use Case** - explain why this is a feature or bug for you 39 | * **Reproduce the error** - if reporting a bug, provide an unambiguous set of steps to reproduce the error. 40 | * **Related issues** - has a similar issue been reported before? 41 | * **Suggest a Fix** - if you can't fix the bug yourself, perhaps you can point to what might be causing 42 | the problem (line of code or commit or general idea) 43 | 44 | ### [Submitting a Pull Request](https://opensource.guide/how-to-contribute/#opening-a-pull-request) 45 | Before you submit your pull request consider the following guidelines: 46 | 47 | * Search [Github](https://github.com/Roche/gitlab-configuration-as-code/pulls) for an open or closed Pull Request 48 | that relates to your submission. 49 | * Fork the repository 50 | * Make your changes in a new git branch 51 | 52 | ```shell 53 | git checkout -b my-branch master 54 | ``` 55 | 56 | * Create your patch, **including appropriate test cases**. 57 | * Follow our [Coding Rules](#rules). 58 | * Ensure that our coding style check passes: 59 | 60 | ```shell 61 | make lint 62 | ``` 63 | 64 | * Ensure that all tests pass 65 | 66 | ```shell 67 | make test 68 | ``` 69 | 70 | * Commit your changes using a descriptive commit message that follows our 71 | [commit message conventions](#commit-message-format). 72 | 73 | ```shell 74 | git commit -a 75 | ``` 76 | 77 | _Note:_ the optional commit `-a` command line option will automatically "add" and "rm" edited files. 78 | 79 | * Push your branch: 80 | 81 | ```shell 82 | git push origin my-branch 83 | ``` 84 | 85 | * In Github, [send a pull request](https://github.com/Roche/gitlab-configuration-as-code/compare) 86 | from your fork to our `master` branch 87 | * There will be default reviewers added. 88 | * If any changes are suggested then 89 | * Make the required updates. 90 | * Re-run tests ensure tests are still passing 91 | 92 | That's it! Thank you for your contribution! 93 | 94 | ## Coding Rules 95 | We use black as code formatter, so you'll need to format your changes using 96 | the [black code formatter](https://github.com/python/black). 97 | 98 | Just run: 99 | ```bash 100 | cd python-gitlab/ 101 | pip3 install --user tox 102 | tox -e black 103 | ``` 104 | to format your code according to our guidelines ([tox](https://tox.readthedocs.io/en/latest/) is required). 105 | 106 | Additionally, `flake8` linter is used to verify code style. It must succeeded 107 | in order to make pull request approved. 108 | 109 | Just run: 110 | ```bash 111 | cd python-gitlab/ 112 | pip3 install --user tox 113 | tox -e flake 114 | ``` 115 | to verify code style according to our guidelines (`tox` is required). 116 | 117 | Before submitting a pull request make sure that the tests still pass with your change. 118 | Unit tests run using Github Actions and passing tests are mandatory 119 | to get merge requests accepted. 120 | 121 | ## Git Commit Guidelines 122 | 123 | We have rules over how our git commit messages must be formatted. 124 | Please ensure to [squash](https://help.github.com/articles/about-git-rebase/#commands-available-while-rebasing) 125 | unnecessary commits so that commit history is clean. 126 | 127 | ### Commit Message Format 128 | Each commit message consists of a **header** and a **body**. 129 | 130 | ``` 131 |
132 | 133 | 134 | ``` 135 | 136 | Any line of the commit message cannot be longer 100 characters! This allows the message to be easier 137 | to read. 138 | 139 | ### Header 140 | The Header contains a succinct description of the change: 141 | 142 | * use the imperative, present tense: "change" not "changed" nor "changes" 143 | * don't capitalize first letter 144 | * no dot (.) at the end 145 | 146 | ### Body 147 | If your change is simple, the Body is optional. 148 | 149 | Just as in the Header, use the imperative, present tense: "change" not "changed" nor "changes". 150 | The Body should include the motivation for the change 151 | 152 | ### Example 153 | For example, here is a good commit message: 154 | 155 | ``` 156 | upgrade to Spring Boot 1.1.7 157 | 158 | upgrade the Maven and Gradle builds to use the new Spring Boot 1.1.7, 159 | see http://spring.io/blog/2014/09/26/spring-boot-1-1-7-released 160 | ``` -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.8-alpine 2 | ARG GCASC_PATH=/opt/gcasc 3 | ARG WORKSPACE=/workspace 4 | 5 | WORKDIR ${GCASC_PATH} 6 | COPY requirements.txt ./ 7 | COPY rtd-requirements.txt ./ 8 | RUN pip --no-cache-dir install -r requirements.txt 9 | 10 | COPY gcasc/ ./ 11 | 12 | RUN ln -s ${GCASC_PATH}/bin/gcasc /usr/local/bin/gcasc 13 | 14 | ENV PYTHONPATH ${GCASC_PATH}/../ 15 | ENV GITLAB_CLIENT_CONFIG_FILE ${WORKSPACE}/gitlab.cfg 16 | ENV GITLAB_CONFIG_FILE ${WORKSPACE}/gitlab.yml 17 | 18 | WORKDIR ${WORKSPACE} 19 | CMD [ "gcasc" ] -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: help clean-pyc clean-build isort lint test build docker-build docker-push docs 2 | 3 | ENV=py37 4 | DOCKER_IMAGE_NAME=gcasc 5 | 6 | help: 7 | @echo "make" 8 | @echo " clean-pyc" 9 | @echo " Remove python artifacts." 10 | @echo " clean-build" 11 | @echo " Remove build artifacts." 12 | @echo " isort" 13 | @echo " Sort import statements." 14 | @echo " lint" 15 | @echo " Check style with flake8." 16 | @echo " test" 17 | @echo " Run tests and produce report in test/out_report.xml" 18 | @echo " build" 19 | @echo ' Build `gcasc` Python package.' 20 | @echo " publish" 21 | @echo ' Publish `gcasc` Python package.' 22 | @echo " docker-build" 23 | @echo ' Build `gcasc` Docker image.' 24 | @echo " docker-push" 25 | @echo ' Publish `gcasc` Docker image.' 26 | 27 | clean-pyc: 28 | find . -name '*.pyc' -exec rm -f {} + 29 | find . -name '*.pyo' -exec rm -f {} + 30 | find . -name 'out_report.xml' -exec rm -f {} + 31 | rm -rf htmlcov .coverage .pytest-cache .tox 32 | 33 | clean-build: 34 | rm -rf build/ 35 | rm -rf dist/ 36 | rm -rf *.egg-info 37 | 38 | install-run-deps: 39 | pip3 install --user -r requirements.txt 40 | 41 | install-test-deps: 42 | pip3 install --user -r test-requirements.txt 43 | 44 | install-deps: install-run-deps install-test-deps 45 | 46 | clean: clean-pyc clean-build 47 | 48 | isort: 49 | sh -c "isort --skip-glob=.tox --recursive . " 50 | 51 | lint: 52 | tox -e flake -e black 53 | 54 | test: clean-pyc 55 | @echo "Running tests on environment: " $(ENV) 56 | tox -e $(ENV) 57 | 58 | docs: clean-build 59 | @echo "Building documentation..." 60 | pip3 install -r requirements.txt 61 | mkdir -p build/docs 62 | cd docs && $(MAKE) html && mv _build/html/* ../build/docs 63 | @echo "Documentation is available in build/docs directory" 64 | 65 | build: clean-build docs 66 | @echo "Building source and binary Wheel distributions..." 67 | pip3 install -r requirements.txt 68 | pip3 install wheel 69 | python3 setup.py sdist bdist_wheel 70 | 71 | publish: build 72 | ifeq ($(strip $(TWINE_USERNAME)),) 73 | @echo "TWINE_USERNAME variable must be provided" 74 | exit -1 75 | endif 76 | ifeq ($(strip $(TWINE_PASSWORD)),) 77 | @echo "TWINE_PASSWORD variable must be provided" 78 | exit -1 79 | endif 80 | @echo "Publishing library to PyPi" 81 | pip3 install twine 82 | twine upload dist/* 83 | @echo "Library published" 84 | 85 | docker-build: 86 | docker build \ 87 | --file=./Dockerfile \ 88 | --tag=$(DOCKER_IMAGE_NAME) ./ 89 | 90 | docker-push: docker-build 91 | ifeq ($(strip $(DOCKER_USERNAME)),) 92 | @echo "DOCKER_USERNAME variable must be provided" 93 | exit -1 94 | endif 95 | ifeq ($(strip $(DOCKER_PASSWORD)),) 96 | @echo "DOCKER_PASSWORD variable must be provided" 97 | exit -1 98 | endif 99 | docker login -u $(DOCKER_USERNAME) -p $(DOCKER_PASSWORD) 100 | docker push $(DOCKER_IMAGE_NAME) 101 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![CI/CD](https://github.com/Roche/gitlab-configuration-as-code/workflows/CI/CD/badge.svg) 2 | [![Docker Pull count](https://img.shields.io/docker/pulls/hoffmannlaroche/gcasc)](https://hub.docker.com/r/hoffmannlaroche/gcasc) 3 | [![PyPI](https://img.shields.io/pypi/v/gitlab-configuration-as-code)](https://pypi.org/project/gitlab-configuration-as-code) 4 | [![Documentation Status](https://readthedocs.org/projects/gitlab-configuration-as-code/badge/?version=latest)](https://gitlab-configuration-as-code.readthedocs.io/en/latest/?badge=latest) 5 | [![Last Commit](https://img.shields.io/github/last-commit/Roche/gitlab-configuration-as-code)]() 6 | [![Python versions](https://img.shields.io/badge/python-3.6%20%7C%203.7%20%7C%203.8-blue)]() 7 | [![License](https://img.shields.io/badge/license-Apache%202.0-blue)](LICENSE) 8 | 9 | # GitLab Configuration as Code (*GCasC*) 10 | 11 | Manage GitLab configuration as code to make it easily manageable, traceable and reproducible. 12 | 13 | ### Table of Contents 14 | 15 | * [Introduction](#introduction) 16 | * [Quick start](#quick-start) 17 | * [Configure client](#configure-client) 18 | * [Prepare GitLab configuration](#prepare-gitlab-configuration) 19 | * [Run GCasC](#run-gcasc) 20 | * [CLI](#cli) 21 | * [Docker image](#docker-image) 22 | * [Examples](#examples) 23 | * [Building](#building) 24 | * [Docker image](#docker-image-1) 25 | * [Python package](#python-package) 26 | * [Testing](#testing) 27 | * [Contribution](#contribution) 28 | * [License](#license) 29 | 30 | ## Introduction 31 | 32 | When configuring your GitLab instance, part of the settings you put in [Omnibus](https://docs.gitlab.com/12.7/omnibus/settings/README.html) 33 | or [Helm Chart](https://docs.gitlab.com/charts/charts/) configuration, and the rest you configure through GitLab UI 34 | or [API](https://docs.gitlab.com/12.7/ee/api/settings.html). Due to tons of configuration options in UI, 35 | making GitLab work as you intend is a complex process. 36 | 37 | We intend to let you automate things you do through now UI in a simple way. The Configuration as Code 38 | has been designed to configure GitLab based on human-readable declarative configuration files written in Yaml. 39 | Writing such a file should be feasible without being a GitLab expert, just translating into code a configuration 40 | process one is used to executing in the web UI. 41 | 42 | _GCasC_ offers a functionality to configure: 43 | * [appearance](https://gitlab-configuration-as-code.readthedocs.io/en/latest/configuration/appearance.html) 44 | * [application settings](https://gitlab-configuration-as-code.readthedocs.io/en/latest/configuration/application_settings.html) 45 | * [features](https://gitlab-configuration-as-code.readthedocs.io/en/latest/configuration/features.html) 46 | * [Instance CI/CD variables](https://gitlab-configuration-as-code.readthedocs.io/en/latest/configuration/instance_variables.html) 47 | * [license](https://gitlab-configuration-as-code.readthedocs.io/en/latest/configuration/license.html) 48 | * ... more coming soon! 49 | 50 | It gives you also a way to: 51 | * include external files or other Yamls using `!include` directive 52 | * inject environment variables into configuration using `!env` directive 53 | into your Yaml configuration. 54 | 55 | Visit [our documentation site](https://gitlab-configuration-as-code.readthedocs.io/) for detailed information on how to use it. 56 | 57 | Configuring your GitLab instance is as simple as this: 58 | ```yaml 59 | appearance: 60 | title: "Your GitLab instance title" 61 | logo: "http://path-to-your-logo/logo.png" 62 | 63 | settings: 64 | elasticsearch: 65 | url: http://elasticsearch.mygitlab.com 66 | username: !env ELASTICSEARCH_USERNAME 67 | password: !env ELASTICSEARCH_PASSWORD 68 | recaptcha_enabled: yes 69 | terms: '# Terms of Service\n\n *GitLab rocks*!!' 70 | plantuml: 71 | enabled: true 72 | url: 'http://plantuml.url' 73 | 74 | instance_variables: 75 | anotherVariable: 'another value' 76 | MY_VARIABLE: 77 | value: !env MY_VARIABLE 78 | protected: false 79 | masked: true 80 | 81 | features: 82 | - name: sourcegraph 83 | value: true 84 | groups: 85 | - mygroup1 86 | projects: 87 | - mygroup2/myproject 88 | users: 89 | - myuser 90 | 91 | license: 92 | starts_at: 2019-11-17 93 | expires_at: 2019-12-17 94 | plan: premium 95 | user_limit: 30 96 | data: !include gitlab.lic 97 | ``` 98 | 99 | **Note:** GCasC supports only Python 3+. Because Python 2.7 end of life is January 1st, 2020 we do not consider support 100 | for Python 2. 101 | 102 | ## Quick start 103 | 104 | Here you will learn how to quickly start with _GCasC_. 105 | 106 | **Important!** Any execution of _GCasC_ may override properties you define in your Yaml files. Don't try it directly 107 | on your production environment. 108 | 109 | Visit [our documentation site](https://gitlab-configuration-as-code.readthedocs.io/) for detailed information on how to use it. 110 | 111 | ### Configure client 112 | 113 | You can configure client in two ways: 114 | 115 | * using configuration file: 116 | ``` 117 | [global] 118 | url = https://gitlab.yourdomain.com 119 | ssl_verify = true 120 | timeout = 5 121 | private_token = 122 | api_version = 4 123 | ``` 124 | By default _GCasC_ is trying to find client configuration file in following paths: 125 | ``` 126 | "/etc/python-gitlab.cfg", 127 | "/etc/gitlab.cfg", 128 | "~/.python-gitlab.cfg", 129 | "~/.gitlab.cfg", 130 | ``` 131 | B 132 | You can provide a path to your configuration file in `GITLAB_CLIENT_CONFIG_FILE` environment variable. 133 | 134 | * using environment variables: 135 | ```bash 136 | GITLAB_CLIENT_URL= # path to GitLab, default: https://gitlab.com 137 | GITLAB_CLIENT_API_VERSION= # GitLab API version, default: 4 138 | GITLAB_CLIENT_TOKEN= # GitLab personal access token 139 | GITLAB_CLIENT_SSL_VERIFY= # Flag if SSL certificate should be verified, default: true 140 | ``` 141 | 142 | You can combine both methods and configuration settings will be searched in the following order: 143 | 144 | * configuration file 145 | * environment variables (due to limitations in `python-gitlab` if using configuration file only `GITLAB_CLIENT_TOKEN` 146 | environment variable will be used) 147 | 148 | Personal access token is mandatory in any client configuration approach and you can configure your it by following 149 | [these instructions](https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html) 150 | 151 | Additionally you can customize HTTP session to enable mutual TLS authentication. To configure this, you should 152 | provide two additional environment variables: 153 | ```bash 154 | GITLAB_CLIENT_CONFIG_FILE= 155 | GITLAB_CLIENT_KEY= 156 | ``` 157 | 158 | ### Prepare GitLab configuration 159 | 160 | GitLab configuration must be defined in Yaml file. You can provide a configuration in a single file, or you can 161 | split it into multiple Yaml files and inject them. 162 | 163 | For information how to prepare GitLab configuration Yaml file visit 164 | [our documentation site](https://gitlab-configuration-as-code.readthedocs.io/en/latest/configuration). 165 | 166 | For `settings` configuration, which defines [Application Settings](https://docs.gitlab.com/12.7/ee/api/settings.html), 167 | the structure is flexible. For example 168 | 169 | ```yaml 170 | settings: 171 | elasticsearch: 172 | url: http://elasticsearch.mygitlab.com 173 | username: elastic_user 174 | password: elastic_password 175 | ``` 176 | 177 | and 178 | B 179 | ```yaml 180 | settings: 181 | elasticsearch_url: http://elasticsearch.mygitlab.com 182 | elasticsearch_username: elastic_user 183 | elasticsearch_password: elastic_password 184 | ``` 185 | are exactly the same and match `elasticsearch_url`, `elasticsearch_username` and `elasticsearch_password` settings. 186 | This means you can flexibly structure your configuration Yaml, where a map child keys are prefixed by parent key (here 187 | `elasticsearch` parent key was a prefix for `url`, `username` and `password` keys). You only need to follow available 188 | [Application Settings](https://docs.gitlab.com/12.7/ee/api/settings.html). 189 | 190 | You can adjust your Yamls by splitting them into multiple or injecting environment variables into certain values using 191 | `!include` or `!env` directives respectively. Example is shown below: 192 | 193 | ```yaml 194 | settings: 195 | elasticsearch: 196 | url: http://elasticsearch.mygitlab.com 197 | username: !env ELASTICSEARCH_USERNAME 198 | password: !env ELASTICSEARCH_PASSWORD 199 | terms: !include tos.md 200 | 201 | license: !include license.yml 202 | ``` 203 | 204 | where: 205 | 206 | * `settings.elasticsearch.username` and `settings.elasticsearch.password` are injected from environment variables 207 | `ELASTICSEARCH_USERNAME` and `ELASTICSEARCH_PASSWORD` respectively 208 | 209 | * `settings.terms` and `license` are injected from `tos.md` plain text file and `license.yml` Yaml file respectively. 210 | In this scenario, your `license.yml` may look like this: 211 | ```yaml 212 | starts_at: 2019-11-17 213 | expires_at: 2019-12-17 214 | plan: premium 215 | user_limit: 30 216 | data: !include gitlab.lic 217 | ``` 218 | 219 | ### Run GCasC 220 | 221 | To run _GCasC_ you can leverage CLI or Docker image. _Docker image is a preferred way_, because it is simple 222 | and does not require from you installing any additional libraries. Also, Docker image was designed that it can be 223 | easily used in your CI/CD pipelines. 224 | 225 | When running locally, you may benefit from running _GCasC_ in TEST mode (default mode is `APPLY`), where no changes 226 | will be applied, but validation will be performed and differences will be logged. Just set `GITLAB_MODE` 227 | environment variable to `TEST`. 228 | ```bash 229 | export GITLAB_MODE=TEST 230 | ``` 231 | 232 | #### CLI 233 | 234 | _GCasC_ library is available in [PyPI](https://pypi.org/project/gitlab-configuration-as-code/). 235 | 236 | To install CLI run `pip install gitlab-configuration-as-code`. Then you can simply execute 237 | ```bash 238 | gcasc 239 | ``` 240 | 241 | //TODO add more information on CLI usage 242 | 243 | Currently, CLI is limited and does not support passing any arguments to it, but behavior can only be configured 244 | using environment variables. Support for CLI arguments may appear in future releases. 245 | 246 | #### Docker image 247 | 248 | Image is available in [Docker Hub](https://hub.docker.com/r/hoffmannlaroche/gcasc). 249 | 250 | _GCasC_ Docker image working directory is `/workspace`. Thus you can quickly launch `gcasc` with: 251 | ```bash 252 | docker run -v $(pwd):/workspace hoffmannlaroche/gcasc 253 | ``` 254 | It will try to find both GitLab client configuration and GitLab configuration in `/workspace` directory. You can modify 255 | the behavior by passing environment variables: 256 | * `GITLAB_CLIENT_CONFIG_FILE` to provide path to GitLab client configuration file 257 | * `GITLAB_CONFIG_FILE` to provide a path to GitLab configuration file 258 | 259 | ```bash 260 | docker run -e GITLAB_CLIENT_CONFIG_FILE=/gitlab/client.cfg -e GITLAB_CONFIG_FILE=/gitlab/config.yml 261 | -v $(pwd):/gitlab hoffmannlaroche/gcasc 262 | ``` 263 | 264 | You can also configure a GitLab client using environment variables. More details about the configuration of GitLab client 265 | are available [in this documentation](https://gitlab-configuration-as-code.readthedocs.io/en/latest/client.html). 266 | 267 | ### Examples 268 | 269 | We provide a few examples to give you a quick starting place to use _GCasC_. They can be found in [`examples`](examples) directory. 270 | 1. [`gitlab.cfg`](examples/gitlab.cfg) is example GitLab client file configuration. 271 | 2. [`basic`](examples/basic/gitlab.yml) is an example GitLab configuration using a single configuration file. 272 | 3. [`environment_variables`](examples/environment_variables) shows how environment variables can be injected 273 | into GitLab configuration file using `!env` directive. 274 | 4. [`modularized`](examples/modularized) shows how you can split single GitLab configuration file into smaller 275 | and inject files containing static text using `!include` directive. 276 | 277 | ## Building 278 | 279 | ### Docker image 280 | 281 | Use `make` to build a basic Docker image quickly. 282 | ```bash 283 | make docker-build 284 | ``` 285 | When using `make` you can additionally pass `DOCKER_IMAGE_NAME` to change default `gcasc:latest` to another image name: 286 | ```bash 287 | make docker-build DOCKER_IMAGE_NAME=mygcasc:1.0 288 | ``` 289 | 290 | To get more control over builds you can use `docker build` directly: 291 | ```bash 292 | docker builds -t gcasc[:TAG] . 293 | ``` 294 | 295 | Dockerfile comes with two build arguments you may use to customize your image by providing `--build-arg` parameter 296 | to `docker build` command: 297 | * `GCASC_PATH` defines the path where _GCasC_ library will be copied. Defaults to `/opt/gcasc`. 298 | * `WORKSPACE` defines a working directory when you run _GCasC_ image. Defaults to `/workspace`. 299 | 300 | ### Python package 301 | 302 | Use `make` to build source distribution (sdist), Wheel binary distribution and Sphinx documentation. 303 | ```bash 304 | make build 305 | ``` 306 | Both source and Wheel distributions will be placed in `dist` directory. Documentation page will be placed 307 | in `build/docs` directory. 308 | 309 | Remember to run tests before building your distribution! 310 | 311 | ## Testing 312 | 313 | Before submitting a pull request make sure that the tests still succeed with your change. 314 | Unit tests run using Github Actions and passing tests are mandatory 315 | to get merge requests accepted. 316 | 317 | You need to install `tox` to run unit tests locally: 318 | 319 | ```bash 320 | # run the unit tests for python 3, python 2, and the flake8 tests: 321 | tox 322 | 323 | # run tests in one environment only: 324 | tox -e py37 325 | 326 | # run flake8 linter and black code formatter 327 | tox -e flake 328 | 329 | # run black code formatter 330 | tox -e black 331 | ``` 332 | 333 | Instead of using `tox` directly, it is recommended to use `make`: 334 | ```bash 335 | # run tests 336 | make test 337 | 338 | # run flake8 linter and black code formatter 339 | make lint 340 | ``` 341 | 342 | ## Contribution 343 | 344 | Everyone is warm welcome to contribute! 345 | 346 | Please make sure to read the [Contributing Guide](CONTRIBUTING.md) and [Code of Conduct](CODE_OF_CONDUCT.md) 347 | before making a pull request. 348 | 349 | ### Contributors 350 | 351 | 352 | 353 | 354 | 361 | 368 | 375 |
355 | 356 | filipowm 357 |
358 | Mateusz 359 | 360 | 		362 | 363 | randombenj 364 |
365 | Benj Fassbind 366 | 367 | 		369 | 370 | 11mariom 371 |
372 | Mariusz Kozakowski 373 | 374 |
376 | 377 | 378 | ## License 379 | 380 | Project is released under [Apache License, Version 2.0 license](LICENSE). 381 | -------------------------------------------------------------------------------- /VERSION: -------------------------------------------------------------------------------- 1 | 0.6.2 2 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line, and also 5 | # from the environment for the first two. 6 | SPHINXOPTS ?= 7 | SPHINXBUILD ?= sphinx-build 8 | SOURCEDIR = . 9 | BUILDDIR = _build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | rm -rf $(BUILDDIR) 21 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 22 | -------------------------------------------------------------------------------- /docs/_static/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Roche/gitlab-configuration-as-code/b8361ba4cf654774443b48e4da05cb12bb5dabb1/docs/_static/.gitkeep -------------------------------------------------------------------------------- /docs/client.md: -------------------------------------------------------------------------------- 1 | # Client Configuration 2 | 3 | GCasC uses a very particular configuration source order that is designed to allow sensible overriding of values. Properties are considered in the following order: 4 | 5 | 1. configuration file 6 | 2. environment variables (due to limitations in `python-gitlab` if using configuration file only `GITLAB_CLIENT_TOKEN` 7 | environment variable will be used) 8 | 9 | **Important!** GitLab does not allow authentication using API with username and password. The preferred approach 10 | is to use personal access tokens. For more about it see [getting personal access token](#getting-personal-access-token). 11 | 12 | ## Configuration file 13 | 14 | Configuration file can have any name, but must contain have following structure (do not omit `[global]` line): 15 | 16 | ```bash 17 | [global] 18 | url = https://gitlab.yourdomain.com 19 | ssl_verify = true # optional 20 | timeout = 5 # optional 21 | private_token = 22 | api_version = 4 # optional, assumes latest 23 | ``` 24 | 25 | By default _GCasC_ is trying to find client configuration file in following paths: 26 | ```bash 27 | /etc/python-gitlab.cfg 28 | /etc/gitlab.cfg 29 | ~/.python-gitlab.cfg 30 | ~/.gitlab.cfg 31 | ``` 32 |   33 | You can provide another path to your configuration file in `GITLAB_CLIENT_CONFIG_FILE` environment variable. 34 | 35 | ## Environment variables 36 | 37 | You can use set up environment variables to configure your API client: 38 | 39 | | **Environment variable** | **Description** | **Default value** | **Example** | 40 | |-----------------------------|----------------------------------------------------------------------------------------------------------------------------------|----------------------------------|---------------------------------| 41 | | `GITLAB_CLIENT_API_VERSION` | Version of GitLab API. Current latest: `4` | `4` | `4` | 42 | | `GITLAB_CLIENT_URL` | URL to GitLab instance. Used only if
`GITLAB_CLIENT_CONFIG_FILE` not provided or invalid. | `https://gitlab.com` | `https://mygitlab.mydomain.com` | 43 | | `GITLAB_CLIENT_SSL_VERIFY` | Flag if SSL certificate of GitLab instance
should be verified. Used only if `GITLAB_CLIENT_CONFIG_FILE`
not provided or invalid. | `true` | `false` | 44 | | `GITLAB_CLIENT_TOKEN` | **Required**. Private token used to access
GitLab API. Used only if `GITLAB_CLIENT_CONFIG_FILE`
not provided or invalid. | | `-uub91Jax13P1iaLkC3za0` | 45 | 46 | ## Getting personal access token 47 | 48 | You **must** have personal access token if you want to use _GCasC_. Personal access token is mandatory in any client 49 | configuration approach. Unfortunately there is no way to configure it via API or get it automatically on instance setup. 50 | Thus you must first have GitLab running (for fresh deploys), then go to the UI and follow 51 | [these instructions](https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html) to get personal access token. 52 | 53 | Recommendation is to limit scopes to minimal set required by the token. Additionally limit the time how long token 54 | is valid. It may not be the most convenient approach for CI/CD pipelines, but gives you additional significant security. 55 | 56 | ## Setting client certificate` 57 | 58 | _GCasC_ allows setting up client certificate in case your GitLab instance requires mutual TLS authentication. 59 | You can configure it same way when using either configuration file or environment variables for client. 60 | 61 | Just provide both of these environment variables. If one of them is missing, error will be raised. 62 | 63 | | **Environment variable** | **Description** | **Example** | 64 | |-----------------------------|-----------------------------------------------------------------------------------------|---------------------------| 65 | | `GITLAB_CLIENT_CERT` | Path to client certificate used for mutual TLS
authentication to access GitLab API. | `/home/myuser/client.crt` | 66 | | `GITLAB_CLIENT_KEY` | Path to client key used for mutual TLS
authentication to access GitLab API. | `/home/myuser/key.pem` | 67 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # Configuration file for the Sphinx documentation builder. 2 | # 3 | # This file only contains a selection of the most common options. For a full 4 | # list see the documentation: 5 | # https://www.sphinx-doc.org/en/master/usage/configuration.html 6 | 7 | # -- Path setup -------------------------------------------------------------- 8 | 9 | # If extensions (or modules to document with autodoc) are in another directory, 10 | # add these directories to sys.path here. If the directory is relative to the 11 | # documentation root, use os.path.abspath to make it absolute, like shown here. 12 | # 13 | import os 14 | 15 | # import sys 16 | # sys.path.insert(0, os.path.abspath('.')) 17 | from recommonmark.transform import AutoStructify 18 | 19 | on_rtd = os.environ.get("READTHEDOCS", None) == "True" 20 | 21 | # -- Project information ----------------------------------------------------- 22 | 23 | project = "GitLab Configuration as Code" 24 | copyright = "2019, Hoffmann-La Roche" 25 | author = "Hoffmann-La Roche" 26 | 27 | # The full version, including alpha/beta/rc tags 28 | release = "0.1" 29 | 30 | # -- General configuration --------------------------------------------------- 31 | 32 | # The master toctree document. 33 | master_doc = "index" 34 | 35 | # Add any Sphinx extension module names here, as strings. They can be 36 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 37 | # ones. 38 | extensions = [ 39 | "sphinx.ext.autodoc", 40 | "sphinx.ext.autosummary", 41 | "sphinx_markdown_tables", 42 | "recommonmark", 43 | ] 44 | 45 | # Add any paths that contain templates here, relative to this directory. 46 | templates_path = ["_templates"] 47 | 48 | # List of patterns, relative to source directory, that match files and 49 | # directories to ignore when looking for source files. 50 | # This pattern also affects html_static_path and html_extra_path. 51 | exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] 52 | 53 | # -- Options for HTML output ------------------------------------------------- 54 | 55 | # The theme to use for HTML and HTML Help pages. See the documentation for 56 | # a list of builtin themes. 57 | # 58 | html_theme = "default" 59 | if not on_rtd: # only import and set the theme if we're building docs locally 60 | try: 61 | import sphinx_rtd_theme 62 | 63 | extensions.append("sphinx_rtd_theme") 64 | html_theme = "sphinx_rtd_theme" 65 | except ImportError: # Theme not found, use default 66 | pass 67 | 68 | # Add any paths that contain custom static files (such as style sheets) here, 69 | # relative to this directory. They are copied after the builtin static files, 70 | # so a file named "default.css" will overwrite the builtin "default.css". 71 | html_static_path = ["_static"] 72 | 73 | # The name of the Pygments (syntax highlighting) style to use. 74 | pygments_style = "sphinx" 75 | 76 | highlight_language = "yaml" 77 | 78 | source_suffix = [".rst", ".md"] 79 | 80 | 81 | def setup(app): 82 | app.add_config_value( 83 | "recommonmark_config", 84 | { 85 | # 'enable_auto_toc_tree': True, 86 | }, 87 | True, 88 | ) 89 | app.add_transform(AutoStructify) 90 | -------------------------------------------------------------------------------- /docs/configuration/appearance.md: -------------------------------------------------------------------------------- 1 | # Appearance 2 | 3 | _GCasC_ allows configuring instance Appearance. Appearance can be configured either through UI (under Apperance in Admin Area) 4 | or API. Using this you can apply branding to your GitLab instance and provide basic information to your users. 5 | 6 | **Reference:** https://docs.gitlab.com/12.7/ee/api/appearance.html 7 | 8 | Appearance structure is flexible. It starts with a root key `appearance`. Then you provide 9 | configuration options as defined in [these docs](https://docs.gitlab.com/12.7/ee/api/appearance.html). For example 10 | 11 | ```yaml 12 | appearance: 13 | title: "GitLab instance title" 14 | description: "Some description of GitLab instance" 15 | header: 16 | logo: "http://path-to-your-logo.com/logo.png" 17 | message: "This is message to show in header" 18 | ``` 19 | 20 | **Note:** Any invalid keys will be discarded, warn message will be presented, but _GCasC_ will continue execution. -------------------------------------------------------------------------------- /docs/configuration/application_settings.md: -------------------------------------------------------------------------------- 1 | # Application Settings 2 | 3 | _GCasC_ allows configuring Application Settings. It consists of plenty of configuration options, that can be set only 4 | through UI or API. They are key to make your GitLab instance work as you intend to. 5 | 6 | **Reference:** https://docs.gitlab.com/12.4/ee/api/settings.html 7 | 8 | Settings the structure is flexible. It starts with a root key `settings`. Then you provide 9 | configuration options as defined in [these docs](https://docs.gitlab.com/12.4/ee/api/settings.html). For example 10 | 11 | ```yaml 12 | settings: 13 | elasticsearch: 14 | url: http://elasticsearch.mygitlab.com 15 | username: elastic_user 16 | password: elastic_password 17 | ``` 18 | 19 | and 20 |   21 | ```yaml 22 | settings: 23 | elasticsearch_url: http://elasticsearch.mygitlab.com 24 | elasticsearch_username: elastic_user 25 | elasticsearch_password: elastic_password 26 | ``` 27 | are exactly the same and match `elasticsearch_url`, `elasticsearch_username` and `elasticsearch_password` settings. 28 | This means you can flexibly structure your configuration Yaml, where a map child keys are prefixed by parent key (here 29 | `elasticsearch` parent key was a prefix for `url`, `username` and `password` keys). Simply: 30 | ```yaml 31 | settings: 32 | prefix1: 33 | prefix2: 34 | value21: 'value21' 35 | value1: 'value1' 36 | prefix1_value2: 'value2' 37 | ``` 38 | will try to configure following properties: `prefix_value1`, `prefix_value2` and `prefix1_prefix2_value21`. 39 | You only need to follow available [Application Settings](https://docs.gitlab.com/12.4/ee/api/settings.html). 40 | 41 | **Note:** Any invalid keys will be discarded, warn message will be presented, but _GCasC_ will continue execution. 42 | 43 | You can adjust your Yamls by splitting them into multiple or injecting environment variables into certain values using 44 | `!include` or `!env` directives respectively. Example is shown below: 45 | 46 | ```yaml 47 | settings: 48 | elasticsearch: !include config/elasticseach.yml 49 | terms: !include tos.md 50 | ``` 51 | 52 | where: 53 | 54 | * `settings.elasticsearch` is injected from file under `./config/elasticsearch.yml` path. Its configuration may look 55 | like this: 56 | ```yaml 57 | url: http://elasticsearch.mygitlab.com 58 | username: !env ELASTICSEARCH_USERNAME 59 | password: !env ELASTICSEARCH_PASSWORD 60 | ``` 61 | Note that here also `ELASTICSEARCH_USERNAME`, `ELASTICSEARCH_PASSWORD` are used to inject username and password 62 | from environment variables 63 | * `settings.terms` is injected from `./tos.md` file 64 | 65 | 66 | -------------------------------------------------------------------------------- /docs/configuration/features.md: -------------------------------------------------------------------------------- 1 | # Instance Feature Flag 2 | 3 | GitLab comes with some functionality configurable using feature flags. 4 | Part of the GitLab functionality is turned off, where to enable it you 5 | need to use API, cause it does not offer UI for setting up feature flags. 6 | 7 | **Reference:** https://docs.gitlab.com/ee/api/features.html 8 | 9 | **Important!** This is authoritative configuration, thus any existing 10 | Feature Flags will be removed and replaced with the ones defined in 11 | config file. If none are defined in config file, existing Feature Flags 12 | will remain untouched. 13 | 14 | Features offered by GitLab are not collected in a single documentation 15 | page, but they are scattered. Please reference to GitLab documentation 16 | for them. Features yaml structure starts with a root key `features` . 17 | It's structure is defined below: 18 | 19 | ```yaml 20 | features: [list] 21 | - name: [string] 22 | value: [bool/int] 23 | feature_group: [string|optional] 24 | groups: [list(string)|optional] 25 | projects: [list(string)|optional] 26 | users: [list(string)|optional] 27 | ``` 28 | 29 | To configure certain feature for a limited set of: 30 | - users, by specifying `users` by their username. 31 | - groups, by specifying `groups` by group short name. 32 | - projects, by specifying `groups` with format `group_name/project_name`. 33 | 34 | Example of complex features configuration: 35 | ```yaml 36 | features: 37 | - name: some_percentage_feature 38 | value: 25 39 | users: 40 | - user1 41 | - user2 42 | - name: some_percentage_feature 43 | value: 50 44 | users: 45 | - myuser 46 | groups: 47 | - mygroup 48 | projects: 49 | - mygroup1/myproject 50 | - mygroup1/myproject2 51 | ``` 52 | 53 | It will configure `some_percentage_feature` with value `25` for users 54 | `user1` and `user2`, while with value `50` for user `myuser`, group 55 | `mygroup` and projects `mygroup1/myproject`, `mygroup1/myproject2`. -------------------------------------------------------------------------------- /docs/configuration/index.md: -------------------------------------------------------------------------------- 1 | # Configuration 2 | 3 | * [Appearance](../appearance.md) 4 | * [Application Settings](../application_settings.md) 5 | * [Feature Flags](../features.md) 6 | * [Instance CI/CD Variables](../instance_variables.md) 7 | * [License](../license.md) 8 | 9 | GitLab configuration is defined in a [YAML](https://docs.ansible.com/ansible/latest/reference_appendices/YAMLSyntax.html). 10 | Providing configuraton for your GitLab instance is as simple as this: 11 | ```yaml 12 | appearance: 13 | title: "Your GitLab instance title" 14 | logo: "http://path-to-your-logo/logo.png" 15 | 16 | settings: 17 | elasticsearch: 18 | url: http://elasticsearch.mygitlab.com 19 | username: !env ELASTICSEARCH_USERNAME 20 | password: !env ELASTICSEARCH_PASSWORD 21 | recaptcha_enabled: yes 22 | terms: !include toc.md 23 | plantuml: 24 | enabled: true 25 | url: 'http://plantuml.url' 26 | 27 | features: 28 | - name: sourcegraph 29 | value: true 30 | groups: 31 | - mygroup1 32 | projects: 33 | - mygroup2/myproject 34 | users: 35 | - myuser 36 | 37 | instance_variables: 38 | MY_VARIABLE: 'value of my instance variable' 39 | ANOTHER_VARIABLE: 40 | value: !env SOME_PASSWORD 41 | masked: true 42 | protected: false 43 | 44 | license: 45 | starts_at: 2019-11-17 46 | expires_at: 2019-12-17 47 | plan: premium 48 | user_limit: 30 49 | data: !include gitlab.lic 50 | ``` 51 | 52 | You can customize where _GCasC_ searches for configuration file or if any changes should be applied on instance 53 | using environment variables. 54 | 55 | | **Environment variable** | **Description** | **Default value** | **Example** | 56 | |-----------------------------|-------------------------------------------------------------------------------------------|----------------------------------|---------------------------------| 57 | | `GITLAB_CONFIG_FILE` | Path to GitLab main configuration. It is
our Configuration as Code entry point. | `./gitlab.yml` | `/home/myuser/gitlabconf.yml` | 58 | | `GITLAB_MODE` | Determine if any changes, when detected,
should be applied. Valid values: `APPLY`, `TEST` | `APPLY` | `TEST` | 59 | 60 | **Yaml directives** 61 | 62 | Custom Yaml directives give you enhanced way of defining your GitLab configuration YAML, where you can 63 | split your configuration into multiple Yaml files or inject environment variables. 64 | 65 | * `!include` to provide path to another Yaml or plain text file which will be included file 66 | under certain key, e.g. 67 | ```yaml 68 | settings: 69 | terms: !include toc.md 70 | elasticsearch: !include config/elasticsearch.yml 71 | ``` 72 | It searches for relative paths in current working directory tree AND in directory tree where GitLab 73 | configuration file is present. 74 | 75 | * `!env` to inject values of environment variables under certain key, e.g. 76 | ```yaml 77 | settings: 78 | elasticsearch_username: !env ELASTICSEARCH_USERNAME 79 | elasticsearch_password: !env ELASTICSEARCH_PASSWORD 80 | ``` 81 | 82 | **Note:** Use `!env` directive to inject secrets into your Yaml. Never put secrets directly in Yaml file! -------------------------------------------------------------------------------- /docs/configuration/instance_variables.md: -------------------------------------------------------------------------------- 1 | # Instance CI/CD Variables 2 | 3 | _GCasC_ allows configuring CI/CD Instance Variables. Instance variables 4 | are useful for no longer needing to manually enter the same credentials 5 | repeatedly for all your projects. Instance-level variables are 6 | available to all projects and groups on the instance. 7 | 8 | **Reference:** https://docs.gitlab.com/ee/api/instance_level_ci_variables.html 9 | 10 | ## Properties 11 | 12 | Instance variables configuration starts with a root key `instance_variables`. 13 | Then you can either define 14 | 1. simple _key-value_ property, where _key_ is a name of variable and 15 | _value_ is its value. 16 | 2. complex property to provide additional variables configuration. 17 | Property _key_ is a name of variables 18 | 19 | Key must be one line, using only letters, numbers, or _ (underscore), with no spaces. 20 | 21 | | **Property** | **Description** | **Default** | 22 | |----------------------------------------------|-----------------------------------------------------------------------------------------------|-------------| 23 | | `instance_variables..value` | Value of the instance variable | | 24 | | `instance_variables..protected` | If `true`, the variable is only available in pipelines that run on protected branches or tags | `false` | 25 | | `instance_variables..masked` | If `true`, variable value is masked in jobs' logs. Value to be masked needs to follow [these requirements](https://docs.gitlab.com/ee/ci/variables/#masked-variable-requirements) | `false` | 26 | | `instance_variables..variable_type` | Type of the variable. Needs to be one of `env_var`, `file` | `env_var` | 27 | 28 | **Note:** You can reference variables in other variables, e.g. you can set 29 | `MY_VARIABLE: 'the other variable is $OTHER_VARIABLE`. 30 | 31 | ## Example 32 | 33 | ```yaml 34 | instance_variables: 35 | MY_VARIABLE: 'value of my instance variable' 36 | ANOTHER_VARIABLE: 37 | value: !env SOME_PASSWORD 38 | masked: true 39 | protected: false 40 | SOME_FILE_VARIABLE: 41 | value: | 42 | long file data 43 | variable_type: file 44 | ``` 45 | -------------------------------------------------------------------------------- /docs/configuration/license.md: -------------------------------------------------------------------------------- 1 | # License 2 | 3 | **Only for Enterprise Edition or gitlab.com. FOSS/Community Edition instance will fail when trying to configure license** 4 | 5 | *GCasC* offers a way to manage your GitLab instance licenses. The clue is that despite license is just a single file, 6 | you need to configure other properties of license so *GCasC* do not upload new (but already used) license with every 7 | execution. That way it is able to recognize that exactly the same license is already in use and skips uploading new one. 8 | Otherwise you could end with very long license history. 9 | 10 | **Reference:** https://docs.gitlab.com/12.4/ee/api/license.html 11 | 12 | ## Properties 13 | 14 | | **Property** | **Description** | **Example** | 15 | |----------------------|------------------------------------------------------------------------------------- |-------------------------| 16 | | `license.starts_at` | Date in format yyyy-MM-dd when license starts | `2019-11-21` | 17 | | `license.expires_at` | Date in format yyyy-MM-dd when license ends | `2019-12-21` | 18 | | `license.plan` | Plan of your GitLab instance license.
Valid values: `starter`, `premium`, `ultimate` | `premium` | 19 | | `license.user_limit` | Number of licensed users | `120` | 20 | | `license.data` | Content of your license file that you
received from GitLab sales | `azhxWFZqb1BsrTVxug...` | 21 | 22 | **Important!** Beware of storing your license in `data` field directly as text. This is insecure and may lead 23 | to leakage of your license. Use `!env` or `!include` directives to inject license to `license.data` field securely from 24 | external source. Also keep your license file itself safe and secure! 25 | 26 | ## Examples 27 | 28 | Full license configuration:: 29 | ```yaml 30 | license: 31 | starts_at: 2019-11-17 32 | expires_at: 2019-12-17 33 | plan: starter 34 | user_limit: 30 35 | data: | 36 | azhxWFZqbk1BOUsrTVxug6AdfzIzWXI1WUVsdWNKRk53V2hiV1FlTUN2TTRS 37 | NkhSVFFhZ3hCajd4bGlLMkhhcUxhd1EySHh2TjJTXG40U3ZNUWM0ZzhqYTE5 38 | T1lcbkJnNERFOVBORkpxK3FsaHZxNFFVSG9GL0NEWWF0elkyOE9SUE41Ny9v 39 | ``` 40 | 41 | Injecting license data from external file:: 42 | ```yaml 43 | license: 44 | starts_at: 2019-11-17 45 | expires_at: 2019-12-17 46 | plan: ultimate 47 | user_limit: 30 48 | data: !include /etc/gitlab/my_gitlab_license.lic 49 | ``` 50 | 51 | Injecting license data from environment variable:: 52 | ```yaml 53 | license: 54 | starts_at: 2019-11-17 55 | expires_at: 2019-12-17 56 | plan: ultimate 57 | user_limit: 30 58 | data: !env GITLAB_LICENSE 59 | ``` 60 | 61 | 62 | -------------------------------------------------------------------------------- /docs/faq.md: -------------------------------------------------------------------------------- 1 | # FAQ 2 | 3 | I'm getting "gcasc.ClientInitializationError: GitLab token was not provided. It must be defined in 4 | `GITLAB_CLIENT_TOKEN` environment variable" 5 | 6 | >It is likely that you provided invalid GitLab client configuration. If you use configuration file, verify 7 | if it has all required configuration parameters and that ``GITLAB_CLIENT_CONFIG_FILE`` environment variable 8 | is set to a path where your config file is. If you use environment variables, verify that you provided 9 | all necessary variables. 10 | See the :ref:`client configuration ` for details. -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. GitLab Configuration as Code documentation master file, created by 2 | sphinx-quickstart on Thu Nov 21 06:49:44 2019. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Welcome to GitLab Configuration as Code's documentation! 7 | ======================================================== 8 | 9 | .. image:: https://travis-ci.org/Roche/gitlab-configuration-as-code.svg?branch=master 10 | :target: https://travis-ci.org/Roche/gitlab-configuration-as-code 11 | :alt: Build Status 12 | .. image:: https://img.shields.io/docker/pulls/hoffmannlaroche/gcasc 13 | :target: https://hub.docker.com/r/hoffmannlaroche/gcasc 14 | :alt: Docker Pull count 15 | .. image:: https://img.shields.io/pypi/v/gitlab-configuration-as-code 16 | :target: https://pypi.org/project/gitlab-configuration-as-code 17 | :alt: PyPI 18 | .. image:: https://img.shields.io/badge/license-Apache%202.0-blue 19 | :alt: License 20 | 21 | When configuring your GitLab instance, part of the settings you put in Omnibus_ or `Helm Chart`__ configuration, 22 | and the rest you configure through GitLab UI or API_. Due to tons of configuration options in UI, making GitLab work 23 | as you intend is a complex process. 24 | 25 | We intend to let you automate things you do through now UI in a simple way. The Configuration as Code 26 | has been designed to configure GitLab based on human-readable declarative configuration files written in Yaml. 27 | Writing such a file should be feasible without being a GitLab expert, just translating into code a configuration 28 | process one is used to executing in the web UI. 29 | 30 | .. _Omnibus: https://docs.gitlab.com/12.4/omnibus/settings/README.html 31 | .. _Helm: https://docs.gitlab.com/charts/charts/ 32 | .. _API: https://docs.gitlab.com/12.4/ee/api/settings.html 33 | __ Helm_ 34 | 35 | Contents: 36 | --------- 37 | .. toctree:: 38 | :maxdepth: 2 39 | 40 | install 41 | usage 42 | client 43 | configuration/index 44 | faq 45 | .. release_notes 46 | .. changelog 47 | -------------------------------------------------------------------------------- /docs/install.md: -------------------------------------------------------------------------------- 1 | # Installation 2 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=. 11 | set BUILDDIR=_build 12 | 13 | if "%1" == "" goto help 14 | 15 | %SPHINXBUILD% >NUL 2>NUL 16 | if errorlevel 9009 ( 17 | echo. 18 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 19 | echo.installed, then set the SPHINXBUILD environment variable to point 20 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 21 | echo.may add the Sphinx directory to PATH. 22 | echo. 23 | echo.If you don't have Sphinx installed, grab it from 24 | echo.http://sphinx-doc.org/ 25 | exit /b 1 26 | ) 27 | 28 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 29 | goto end 30 | 31 | :help 32 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 33 | 34 | :end 35 | popd 36 | -------------------------------------------------------------------------------- /docs/usage.md: -------------------------------------------------------------------------------- 1 | # Usage 2 | 3 | ## Docker image 4 | 5 | Image is available in [Docker Hub](https://hub.docker.com/r/hoffmannlaroche/gcasc). 6 | 7 | _GCasC_ Docker image working directory is `/workspace`. Thus you can quickly launch `gcasc` with: 8 | ```bash 9 | docker run -v $(pwd):/workspace hoffmannlaroche/gcasc 10 | ``` 11 | It will try to find both GitLab client configuration and GitLab configuration in `/workspace` directory. You can modify 12 | the behavior by passing environment variables: 13 | * `GITLAB_CLIENT_CONFIG_FILE` to provide path to GitLab client configuration file 14 | * `GITLAB_CONFIG_FILE` to provide a path to GitLab configuration file 15 | 16 | ```bash 17 | docker run 18 | -e GITLAB_CLIENT_CONFIG_FILE=/gitlab/client.cfg 19 | -e GITLAB_CONFIG_FILE=/gitlab/config.yml 20 | -v $(pwd):/gitlab 21 | hoffmannlaroche/gcasc 22 | ``` 23 | 24 | You can also configure a GitLab client using environment variables. More details about the configuration of 25 | GitLab client is [here](client.md). 26 | 27 | 28 | ## CLI 29 | 30 | 31 | -------------------------------------------------------------------------------- /examples/basic/gitlab.yml: -------------------------------------------------------------------------------- 1 | # Reference: https://docs.gitlab.com/12.7/ee/api/settings.html 2 | settings: 3 | asset_proxy: 4 | enabled: False 5 | external_authorization_service: 6 | enabled: True 7 | default_label: 'some label' 8 | timeout: 55 9 | url: 'https://some.external.auth.service.url' 10 | recaptcha_enabled: yes 11 | terms: '# Terms of Service\n\n *GitLab rocks*!!' 12 | plantuml: 13 | enabled: True 14 | url: 'http://plantuml.url' 15 | 16 | instance_variables: 17 | MY_VARIABLE: 18 | value: 'myvariable@value' 19 | protected: false 20 | masked: true 21 | ANOTHER_VARIABLE: 'another value' 22 | 23 | license: 24 | starts_at: 2019-11-17 25 | expires_at: 2019-12-17 26 | plan: premium 27 | user_limit: 30 28 | data: | 29 | azhxWFZqbk1BOUsrTVxuVTlLazIzWXI1WUVsdWNKRk53V2hiV1FlTUN2TTRS 30 | NkhSVFFhZ3hCajd4bGlLMkhhcUxhd1EySHh2TjJTXG40U3ZNUWM0ZzhqYTE5 31 | WDg1L1NqYjk3dFZOcnhOWEtDNmhQbHNOVGx6RkU1Q3Z3Umh1alBRdVJjM0dy 32 | T1lcbkJnNERFOVBORkpxK3FsaHZxNFFVSG9GL0NEWWF0elkyOE9SUE41Ny9v 33 | WWo0a3JMQXFNTW85MWpjZmV3b1xuU0NsZmM3UTEzZ3VQMVVhNHJaZ2lVOFgr 34 | cGNYMFNMU1Y1a0x4UkpNMnhIOWlLZ3NFTzlRYTZIUU4wZlZEXG5Lc0ZrV2Zu 35 | TThXdEc2UVhabllPNkgwV1VhbW9ybm9YZ3hocU9Lci9CMXRRR1paNUpzT2Rz 36 | ZURxbGtRK0lcbnJZMEFBU1ZHM3hCWEZiN1QxREVVcHBLTEgyTlpaaUdMRGx1 37 | MHgxeWtZN09taXVoZlpubGtCenhleWh0K1xud0d0c1kyZGRuYnNrMzU0b0xD 38 | R3hkWVVGdW5DWHh3NVczd2FpR0dHMTBQM2ZQenVGK0pGT25OcEN4RHg1XG5z 39 | bFNWbGE5bXhhWmM5V0Rua1hRPVxuIiwia2V5IjoidmhhWmlEZXJSam9aemdh 40 | WXExWGFLL2NFVVZwUGY5Wm02VmIvSDh0VSt5MXYrcnZJN2pqSjh2VzhJbXVU 41 | XG5EN2hmdytPdGltNmZEZHdKa2kyYjZVYUhEczFZcVJISFFGd0tSb0JRZDNX 42 | 43 | appearance: 44 | title: "Your GitLab instance title" 45 | logo: "http://path-to-your-logo/logo.png" -------------------------------------------------------------------------------- /examples/environment_variables/external_auth.yml: -------------------------------------------------------------------------------- 1 | enabled: True 2 | default_label: 'some label' 3 | timeout: 55 4 | url: !env EXTERNAL_AUTH_URL -------------------------------------------------------------------------------- /examples/environment_variables/gitlab.yml: -------------------------------------------------------------------------------- 1 | # Reference: https://docs.gitlab.com/12.4/ee/api/settings.html 2 | settings: 3 | asset_proxy: 4 | enabled: False 5 | external_authorization_service: !include external_auth.yml 6 | recaptcha_enabled: !env RECAPTCHA_ENABLED:True # True if RECAPTCHA_ENABLED is not found in environment 7 | repository_storages: 8 | - !env REPO_STORAGE_1 9 | - !env REPO_STORAGE_2 10 | - /some/defined/repo/path 11 | plantuml: 12 | enabled: true 13 | url: 'http://plantuml.url' 14 | 15 | instance_variables: 16 | MY_VARIABLE: 17 | value: !env MY_VAR 18 | protected: true 19 | 20 | license: 21 | starts_at: 2019-11-17 22 | expires_at: 2019-12-17 23 | plan: premium 24 | user_limit: 30 25 | data: !env GITLAB_LICENSE -------------------------------------------------------------------------------- /examples/gitlab.cfg: -------------------------------------------------------------------------------- 1 | [global] 2 | url = https://gitlab.yourdomain.com 3 | ssl_verify = true 4 | timeout = 5 5 | private_token = your_personal_access_token 6 | api_version = 4 -------------------------------------------------------------------------------- /examples/modularized/appearance.yml: -------------------------------------------------------------------------------- 1 | # Reference: https://docs.gitlab.com/12.7/ee/api/appearance.html 2 | 3 | title: "Your GitLab instance title" 4 | logo: "http://path-to-your-logo/logo.png" -------------------------------------------------------------------------------- /examples/modularized/gitlab.yml: -------------------------------------------------------------------------------- 1 | # Reference: https://docs.gitlab.com/12.7/ee/api/settings.html 2 | 3 | settings: !include settings/settings.yml 4 | license: !include license.yml 5 | appearance: !include appearance.yml -------------------------------------------------------------------------------- /examples/modularized/license.yml: -------------------------------------------------------------------------------- 1 | starts_at: 2019-11-17 2 | expires_at: 2019-12-17 3 | plan: premium 4 | user_limit: 30 5 | data: | 6 | azhxWFZqbk1BOUsrTVxuVTlLazIzWXI1WUVsdWNKRk53V2hiV1FlTUN2TTRS 7 | NkhSVFFhZ3hCajd4bGlLMkhhcUxhd1EySHh2TjJTXG40U3ZNUWM0ZzhqYTE5 8 | WDg1L1NqYjk3dFZOcnhOWEtDNmhQbHNOVGx6RkU1Q3Z3Umh1alBRdVJjM0dy 9 | T1lcbkJnNERFOVBORkpxK3FsaHZxNFFVSG9GL0NEWWF0elkyOE9SUE41Ny9v 10 | WWo0a3JMQXFNTW85MWpjZmV3b1xuU0NsZmM3UTEzZ3VQMVVhNHJaZ2lVOFgr 11 | cGNYMFNMU1Y1a0x4UkpNMnhIOWlLZ3NFTzlRYTZIUU4wZlZEXG5Lc0ZrV2Zu 12 | TThXdEc2UVhabllPNkgwV1VhbW9ybm9YZ3hocU9Lci9CMXRRR1paNUpzT2Rz 13 | ZURxbGtRK0lcbnJZMEFBU1ZHM3hCWEZiN1QxREVVcHBLTEgyTlpaaUdMRGx1 14 | MHgxeWtZN09taXVoZlpubGtCenhleWh0K1xud0d0c1kyZGRuYnNrMzU0b0xD 15 | R3hkWVVGdW5DWHh3NVczd2FpR0dHMTBQM2ZQenVGK0pGT25OcEN4RHg1XG5z 16 | bFNWbGE5bXhhWmM5V0Rua1hRPVxuIiwia2V5IjoidmhhWmlEZXJSam9aemdh 17 | WXExWGFLL2NFVVZwUGY5Wm02VmIvSDh0VSt5MXYrcnZJN2pqSjh2VzhJbXVU 18 | XG5EN2hmdytPdGltNmZEZHdKa2kyYjZVYUhEczFZcVJISFFGd0tSb0JRZDNX -------------------------------------------------------------------------------- /examples/modularized/settings/asset_proxy.yml: -------------------------------------------------------------------------------- 1 | enabled: False -------------------------------------------------------------------------------- /examples/modularized/settings/external_auth.yml: -------------------------------------------------------------------------------- 1 | enabled: True 2 | default_label: 'some label' 3 | timeout: 55 4 | url: 'https://some.external.auth.service.url' 5 | -------------------------------------------------------------------------------- /examples/modularized/settings/settings.yml: -------------------------------------------------------------------------------- 1 | # Reference: https://docs.gitlab.com/12.4/ee/api/settings.html 2 | 3 | asset_proxy: !include asset_proxy.yml 4 | external_authorization_service: !include external_auth.yml 5 | recaptcha_enabled: yes 6 | terms: !include tos.md 7 | plantuml: 8 | enabled: True 9 | url: 'http://plantuml.url' -------------------------------------------------------------------------------- /examples/modularized/settings/tos.md: -------------------------------------------------------------------------------- 1 | # This is Terms of Service example 2 | 3 | - term 1 4 | - term 2 5 | 6 | Some long text line 1 7 | some long text line 2 -------------------------------------------------------------------------------- /gcasc/__init__.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | from pathlib import Path 4 | 5 | import gitlab 6 | import requests 7 | from gitlab import GitlabError 8 | 9 | import gcasc.exceptions as exceptions 10 | import gcasc.utils.os as uos 11 | from gcasc.exceptions import ( 12 | ClientInitializationException, 13 | GcascException, 14 | ) 15 | 16 | from .appearance import AppearanceConfigurer 17 | from .base import Mode 18 | from .config import GitlabConfiguration 19 | from .features import FeaturesConfigurer 20 | from .instance_variables import InstanceVariablesConfigurer 21 | from .license import LicenseConfigurer 22 | from .settings import SettingsConfigurer 23 | from .utils import logger as logging 24 | 25 | GITLAB_CLIENT_CONFIG_FILE = ["GITLAB_CLIENT_CONFIG", "GITLAB_CLIENT_CONFIG_FILE"] 26 | GITLAB_CLIENT_CERTIFICATE = ["GITLAB_CLIENT_CERT", "GITLAB_CLIENT_CERTIFICATE"] 27 | GITLAB_CLIENT_KEY = "GITLAB_CLIENT_KEY" 28 | GITLAB_CLIENT_URL = "GITLAB_CLIENT_URL" 29 | GITLAB_CLIENT_API_VERSION = "GITLAB_CLIENT_API_VERSION" 30 | GITLAB_CLIENT_TOKEN = "GITLAB_CLIENT_TOKEN" 31 | GITLAB_CLIENT_SSL_VERIFY = "GITLAB_CLIENT_SSL_VERIFY" 32 | GITLAB_CONFIG_FILE = ["GITLAB_CONFIG_FILE", "GITLAB_CONFIG_PATH"] 33 | 34 | GITLAB_CONFIG_FILE_DEFAULT_PATHS = [ 35 | "/etc/python-gitlab.cfg", 36 | "/etc/gitlab.cfg", 37 | "~/.python-gitlab.cfg", 38 | "~/.gitlab.cfg", 39 | ] 40 | 41 | GITLAB_MODE = "GITLAB_MODE" 42 | 43 | logger = logging.get_logger() 44 | 45 | configurers = [ 46 | SettingsConfigurer, 47 | LicenseConfigurer, 48 | AppearanceConfigurer, 49 | FeaturesConfigurer, 50 | InstanceVariablesConfigurer, 51 | ] 52 | 53 | 54 | def create_all_configurers(gitlab, config, mode): 55 | return map( 56 | lambda configurer: configurer(gitlab, config.get(configurer._NAME), mode), 57 | configurers, 58 | ) 59 | 60 | 61 | class GitlabConfigurationAsCode(object): 62 | def __init__(self, configurers=None): 63 | # type: ()->GitlabConfigurationAsCode 64 | self.mode = Mode[uos.get_env_or_else(GITLAB_MODE, Mode.APPLY.name)] 65 | if self.mode == Mode.TEST: 66 | logger.info("TEST MODE ENABLED. NO CHANGES WILL BE APPLIED") 67 | path = uos.get_env_or_else( 68 | GITLAB_CONFIG_FILE, "{0}/gitlab.yml".format(os.getcwd()) 69 | ) 70 | self.gitlab = init_gitlab_client() 71 | self.config = GitlabConfiguration.from_file(path) 72 | self.configurers = ( 73 | configurers 74 | if configurers 75 | else create_all_configurers(self.gitlab, self.config, self.mode) 76 | ) 77 | version, revision = self.gitlab.version() 78 | logger.info("GitLab version: %s, revision: %s", version, revision) 79 | 80 | def configure(self, target=None): 81 | if target is None: 82 | for configurer in self.configurers: 83 | try: 84 | self._configure(configurer) 85 | except GcascException as exc: 86 | exceptions.handle_gcasc_exception(exc, logger) 87 | sys.exit(1) 88 | except GitlabError as exc: 89 | exceptions.handle_gitlab_exception(exc, configurer.logger) 90 | sys.exit(1) 91 | 92 | def _configure(self, configurer): 93 | if configurer.config is not None: 94 | if self.mode != Mode.TEST_SKIP_VALIDATION: 95 | configurer.validate() 96 | configurer.configure() 97 | else: 98 | logger.debug( 99 | "Skipping configurer %s because it does not have any configuration to apply", 100 | configurer.__class__.__name__, 101 | ) 102 | 103 | 104 | def init_gitlab(): 105 | # type: ()->gitlab.Gitlab 106 | config_path = __find_gitlab_connection_config_file() 107 | config = ( 108 | gitlab.config.GitlabConfigParser(gitlab_id="global", config_files=[config_path]) 109 | if config_path is not None 110 | else None 111 | ) 112 | 113 | token = getattr(config, "private_token", None) or uos.get_env_or_else( 114 | GITLAB_CLIENT_TOKEN 115 | ) 116 | if token is None: 117 | raise ClientInitializationException( 118 | "GitLab token was not provided. It must be defined in {0} environment variable or config file".format( 119 | GITLAB_CLIENT_TOKEN 120 | ) 121 | ) 122 | 123 | url = getattr(config, "url", None) or uos.get_env_or_else( 124 | GITLAB_CLIENT_URL, "https://gitlab.com" 125 | ) 126 | # ssl_verify is always set by GitlabConfigParser, using inline if to handle `false` value read from config file 127 | ssl_verify = ( 128 | config.ssl_verify 129 | if config 130 | else uos.get_env_or_else(GITLAB_CLIENT_SSL_VERIFY, True) 131 | ) 132 | api_version = getattr(config, "api_version", None) or uos.get_env_or_else( 133 | GITLAB_CLIENT_API_VERSION, "4" 134 | ) 135 | 136 | return gitlab.Gitlab( 137 | url=url, private_token=token, ssl_verify=ssl_verify, api_version=api_version 138 | ) 139 | 140 | 141 | def init_gitlab_client(): 142 | # type: ()->gitlab.Gitlab 143 | logger.info("Initializing GitLab client") 144 | logger.info("Trying to initialize GitLab client...") 145 | client = init_gitlab() 146 | 147 | if client is None: 148 | raise ClientInitializationException( 149 | "Unable to initialize GitLab client due to missing configuration either in " 150 | "config file or environment vars" 151 | ) 152 | 153 | __init_session(client) 154 | return client 155 | 156 | 157 | def __find_gitlab_connection_config_file(): 158 | # type: ()->str 159 | config_path = uos.get_env_or_else(GITLAB_CLIENT_CONFIG_FILE) 160 | if config_path is not None: 161 | if not __check_file_exists(config_path, "GitLab Client"): 162 | logger.error( 163 | "Configuration file was not found under path %s, which was defined in %s env variable." 164 | "Provide path to existing file or remove this variable and configure client" 165 | "using environment variables instead of configuration file", 166 | config_path, 167 | GITLAB_CLIENT_CONFIG_FILE, 168 | ) 169 | return None 170 | else: 171 | for path in GITLAB_CONFIG_FILE_DEFAULT_PATHS: 172 | if __check_file_exists(path, "GitLab Client"): 173 | config_path = path 174 | break 175 | return config_path 176 | 177 | 178 | def __check_file_exists(path, file_context=""): 179 | # type: (str, str)->bool 180 | config = Path(path) 181 | if not config.exists(): 182 | logger.error( 183 | "[%s] File under %a does not exist. Provide valid path.", file_context, path 184 | ) 185 | return False 186 | 187 | if config.is_dir(): 188 | logger.error( 189 | "[%s] Directory was found under %s instead of file. Provide valid path to file.", 190 | file_context, 191 | path, 192 | ) 193 | return False 194 | return True 195 | 196 | 197 | def __init_session(gitlab): 198 | # type: (gitlab.Gitlab)->() 199 | certificate = uos.get_env_or_else(GITLAB_CLIENT_CERTIFICATE) 200 | key = uos.get_env_or_else(GITLAB_CLIENT_KEY) 201 | if certificate is None and key is None: 202 | return 203 | elif certificate is None and key is not None: 204 | pass 205 | elif certificate is not None and key is None: 206 | pass 207 | else: 208 | check_config = __check_file_exists( 209 | certificate, "Client Certificate" 210 | ) and __check_file_exists(key, "Client Key") 211 | if not check_config: 212 | raise ClientInitializationException( 213 | "GitLab client authentication env vars were provided, but point to incorrect file(s)" 214 | ) 215 | session = requests.Session() 216 | session.cert = (certificate, key) 217 | gitlab.session = session 218 | -------------------------------------------------------------------------------- /gcasc/appearance.py: -------------------------------------------------------------------------------- 1 | from .base import Mode, UpdateOnlyConfigurer 2 | 3 | 4 | class AppearanceConfigurer(UpdateOnlyConfigurer): 5 | _NAME = "appearance" 6 | 7 | def __init__(self, gitlab, settings, mode=Mode.APPLY): 8 | super().__init__("Appearance", gitlab, settings, mode=mode) 9 | 10 | def _load(self): 11 | return self.gitlab.appearance.get() 12 | -------------------------------------------------------------------------------- /gcasc/base.py: -------------------------------------------------------------------------------- 1 | import collections 2 | from abc import ABC 3 | from enum import Enum 4 | 5 | import jsonschema 6 | import yaml 7 | from gitlab import Gitlab 8 | from jsonschema import draft7_format_checker 9 | 10 | from . import GcascException 11 | from .exceptions import ValidationException 12 | from .utils import logger as logging 13 | from .utils.validators import ValidationResult, create_message 14 | 15 | 16 | class Mode(Enum): 17 | TEST_SKIP_VALIDATION = -2 18 | TEST = -1 19 | APPLY = 0 20 | 21 | 22 | class Configurer(ABC): 23 | _NAME = None 24 | _SCHEMA = None 25 | 26 | def __init__( 27 | self, gitlab, config, mode=Mode.APPLY 28 | ): # type: (Gitlab, dict, Mode)->any 29 | 30 | if not self._NAME: 31 | raise GcascException( 32 | "Class property _NAME must be defined! It will be expected " 33 | "to exist in configuration file as a key" 34 | ) 35 | self.logger = logging.get_logger(self._NAME) 36 | self.gitlab = gitlab 37 | self.config = config 38 | self.mode = mode 39 | 40 | def configure(self): 41 | pass 42 | 43 | def _validate(self, result): # type: (ValidationResult) -> () 44 | pass 45 | 46 | def _get(self, property): 47 | return self.config.get(property) 48 | 49 | def __apply_schema_error(self, err, result): 50 | message = create_message(err) 51 | path = list(collections.deque(err.absolute_path)) 52 | if len(path) == 0: 53 | # assume that for empty path this is objet property, thus value needs to be included in path 54 | path.append(err.instance) 55 | result.add(message, path=path) 56 | 57 | def __validate_schema(self, result): # type: (ValidationResult) -> () 58 | self.logger.debug("Validating configuration schema") 59 | schema = yaml.safe_load(self._SCHEMA) 60 | validator = jsonschema.Draft7Validator( 61 | schema, format_checker=draft7_format_checker 62 | ) 63 | for err in sorted( 64 | validator.iter_errors(self.config), key=lambda error: error.absolute_path 65 | ): 66 | if err.validator in ["allOf", "anyOf", "oneOf", "patternProperties"]: 67 | [self.__apply_schema_error(ctx_err, result) for ctx_err in err.context] 68 | else: 69 | self.__apply_schema_error(err, result) 70 | 71 | def validate(self): 72 | self.logger.debug("Validating provided configuration") 73 | if self.gitlab is None: 74 | raise GcascException("GitLab client is not initialized") 75 | if self.config is not None: 76 | result = ValidationResult(self._NAME) 77 | if self._SCHEMA is not None and len(self._SCHEMA) > 0: 78 | self.__validate_schema(result) 79 | self._validate(result) 80 | if result and result.has_errors(): 81 | if self.logger.is_debug_enabled(): 82 | self.logger.debug("Validation errors found:") 83 | result.iterate(lambda message: self.logger.debug(message)) 84 | error = ValidationException.from_validation_result(result) 85 | raise error 86 | 87 | 88 | class UpdateOnlyConfigurer(Configurer): 89 | def __init__(self, name, gitlab, config, mode=Mode.APPLY): 90 | self.name = name 91 | self.logger = logging.get_logger(name) 92 | super().__init__(gitlab, config, mode=mode) 93 | 94 | def _save(self, data): 95 | data.save() 96 | 97 | def _load(self): 98 | raise NotImplementedError("") 99 | 100 | def configure(self): 101 | self.logger.info("Configuring %s", self.name) 102 | self.logger.debug("Loading data") 103 | data = self._load() 104 | self.logger.debug("Data loaded. Updating setttings...") 105 | changes = self._update_setting(data, self.config) 106 | self.logger.info("Found %s changed values", changes) 107 | if changes != 0: 108 | self.logger.info("Applying changes...") 109 | if self.mode == Mode.APPLY: 110 | self._save(data) 111 | else: 112 | self.logger.info("No changes will be applied due to test mode enabled") 113 | else: 114 | self.logger.info("Nothing to do") 115 | return data 116 | 117 | def _update_setting( 118 | self, current, new, changes=0, prefix="" 119 | ): # type: (dict, dict, int, str)->int 120 | for key, value in new.items(): 121 | if isinstance(value, dict): 122 | changes += self._update_setting( 123 | current, value, changes, "{0}{1}_".format(prefix, key) 124 | ) 125 | continue 126 | 127 | prefixed_key = "{0}{1}".format(prefix, key) 128 | 129 | self.logger.debug("Checking %s", prefixed_key) 130 | if hasattr(current, prefixed_key): 131 | current_value = getattr(current, prefixed_key) 132 | if current_value != value: 133 | changes += 1 134 | if current_value is None: 135 | self.logger.info("Set: %s = %s", prefixed_key, value) 136 | elif value is None: 137 | self.logger.info("Unset: %s = %s", prefixed_key, current_value) 138 | else: 139 | self.logger.info( 140 | "Updated %s: %s => %s", prefixed_key, current_value, value 141 | ) 142 | if self.mode == Mode.APPLY: 143 | setattr(current, prefixed_key, value) 144 | else: 145 | self.logger.warn( 146 | "Invalid configuration option: %s. Skipping...", prefixed_key 147 | ) 148 | return changes 149 | -------------------------------------------------------------------------------- /gcasc/bin/gcasc: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | from gcasc import GitlabConfigurationAsCode 3 | 4 | 5 | def welcome(): 6 | print(r""" 7 | _____ _____ _____ 8 | | __ \/ __ \ / __ \ 9 | | | \/| / \/ __ _ ___ | / \/ 10 | | | __ | | / _` |/ __|| | 11 | | |_\ \| \__/\| (_| |\__ \| \__/\ 12 | \____/ \____/ \__,_||___/ \____/ 13 | """, flush=True) 14 | 15 | 16 | if __name__ == '__main__': 17 | welcome() 18 | GitlabConfigurationAsCode().configure() 19 | -------------------------------------------------------------------------------- /gcasc/config.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import yaml 4 | 5 | from gcasc import GcascException 6 | from gcasc.utils import os as uos 7 | from gcasc.utils.yaml_include import YamlIncluderConstructor 8 | 9 | # includer relative to current path from which script is executed 10 | YamlIncluderConstructor.add_to_loader_class( 11 | loader_class=yaml.FullLoader, base_dir=os.path.dirname(os.path.realpath(__file__)) 12 | ) 13 | 14 | # includer relative to GitLab configuration file path 15 | config_path = uos.get_env_or_else( 16 | ["GITLAB_CONFIG_FILE", "GITLAB_CONFIG_PATH"], "{0}/x".format(os.getcwd()) 17 | ).split("/") 18 | del config_path[-1] 19 | YamlIncluderConstructor.add_to_loader_class( 20 | loader_class=yaml.FullLoader, base_dir="/".join(config_path) 21 | ) 22 | 23 | 24 | class GitlabConfiguration(object): 25 | def __init__(self, config): 26 | # type: (dict)->GitlabConfiguration 27 | if config is None: 28 | raise GcascException("GitLab configuration is empty") 29 | 30 | if not isinstance(config, dict): 31 | raise GcascException("Configuration provided must be of dictionary type") 32 | 33 | self.config = config 34 | 35 | def get(self, configuration): 36 | return self.config.get(configuration) 37 | 38 | @staticmethod 39 | def from_file(path): 40 | # type: (str)->GitlabConfiguration 41 | with open(path) as file: 42 | data = yaml.load(file, Loader=yaml.FullLoader) 43 | return GitlabConfiguration(data) 44 | -------------------------------------------------------------------------------- /gcasc/exceptions.py: -------------------------------------------------------------------------------- 1 | from gitlab import ( 2 | GitlabAuthenticationError, 3 | GitlabCreateError, 4 | GitlabDeleteError, 5 | GitlabError, 6 | GitlabGetError, 7 | GitlabLicenseError, 8 | GitlabListError, 9 | GitlabSetError, 10 | GitlabUpdateError, 11 | ) 12 | 13 | from gcasc.utils.logger import Logger 14 | 15 | 16 | class GcascException(Exception): 17 | pass 18 | 19 | 20 | class ClientInitializationException(GcascException): 21 | pass 22 | 23 | 24 | class ValidationException(GcascException): 25 | def __init__(self, message, result=None): 26 | super().__init__(message) 27 | self.result = result 28 | 29 | @staticmethod 30 | def from_validation_result(validation_result): 31 | if validation_result is None or not validation_result.has_errors(): 32 | return ValidationException("Validation failed, but no errors were provided") 33 | return ValidationException( 34 | "Configuration validation failed with following errors", 35 | result=validation_result, 36 | ) 37 | 38 | 39 | def _log_exception(ex, logger): 40 | logger.error(ex) 41 | 42 | 43 | def _handle_validation_exception( 44 | exc, logger 45 | ): # type: (ValidationException, Logger) -> () 46 | _log_exception(exc, logger) 47 | exc.result.iterate(lambda message: logger.error(message)) 48 | 49 | 50 | def handle_gcasc_exception(exc, logger): # type: (GcascException, Logger) -> () 51 | switch = {ValidationException: _handle_validation_exception} 52 | exc_type = type(exc) 53 | switch.get(exc_type, _log_exception)(exc, logger) 54 | 55 | 56 | def handle_gitlab_exception(exc, logger): # type: (GitlabError, Logger) -> () 57 | switch = { 58 | GitlabAuthenticationError: "Authentication to GitLab failed", 59 | GitlabListError: "Unable to list GitLab resources", 60 | GitlabGetError: "Unable to get GitLab resource", 61 | GitlabCreateError: "GitLab resource creation failed", 62 | GitlabUpdateError: "GitLab resource update failed", 63 | GitlabDeleteError: "GitLab resource deletion failed", 64 | GitlabSetError: "GitLab resource property change failed", 65 | GitlabLicenseError: "Provided licence is invalid", 66 | } 67 | exc_type = type(exc) 68 | message = switch.get(exc_type, "Error occurred while communicating with GitLab") 69 | logger.error(f"{message}\nHTTP {exc.response_code}, {exc.error_message}") 70 | -------------------------------------------------------------------------------- /gcasc/features.py: -------------------------------------------------------------------------------- 1 | from .base import Configurer, Mode 2 | from .utils import logger 3 | 4 | logger = logger.get_logger("Features") 5 | 6 | 7 | class FeaturesConfigurer(Configurer): 8 | _NAME = "features" 9 | _SCHEMA = """ 10 | type: array 11 | items: 12 | - type: object 13 | required: 14 | - name 15 | - value 16 | properties: 17 | name: 18 | type: string 19 | value: 20 | type: 21 | - string 22 | - boolean 23 | - number 24 | projects: 25 | type: array 26 | items: 27 | type: string 28 | users: 29 | type: array 30 | items: 31 | type: string 32 | groups: 33 | type: array 34 | items: 35 | type: string 36 | additionalProperties: false 37 | """ 38 | 39 | def __init__( 40 | self, gitlab, features, mode=Mode.APPLY 41 | ): # type: (gitlab.Gitlab, dict, Mode)->FeaturesConfigurer 42 | super().__init__(gitlab, features, mode=mode) 43 | 44 | def configure(self): 45 | logger.info("Configuring GitLab Features") 46 | self.__remove_existing() 47 | 48 | for feature in self.config: 49 | name = feature["name"] 50 | value = feature["value"] 51 | if self.mode == Mode.APPLY: 52 | self.__apply(name, value, feature) 53 | logger.info("Configured: %s => %s", name, value) 54 | return self.gitlab.features.list() 55 | 56 | def __set_canary( 57 | self, canary_name, canaries, feature_name, feature_value, feature_group 58 | ): 59 | if canaries: 60 | for canary_value in canaries: 61 | logger.info("Configuring canary: %s => %s", canary_name, canary_value) 62 | self.gitlab.features.set( 63 | feature_name, 64 | feature_value, 65 | feature_group=feature_group, 66 | **{canary_name: canary_value} 67 | ) 68 | return True 69 | return False 70 | 71 | def __apply(self, name, value, feature): 72 | feature_group = feature.get("feature_group") 73 | users = feature.get("users") 74 | is_set = self.__set_canary("user", users, name, value, feature_group) 75 | groups = feature.get("groups") 76 | is_set = ( 77 | self.__set_canary("group", groups, name, value, feature_group) or is_set 78 | ) 79 | projects = feature.get("projects") 80 | is_set = ( 81 | self.__set_canary("project", projects, name, value, feature_group) or is_set 82 | ) 83 | if not is_set: # set feature globally 84 | self.gitlab.features.set(name, value, feature_group=feature_group) 85 | 86 | def __remove_existing(self): 87 | if self.mode == Mode.APPLY: 88 | features = self.gitlab.features.list() 89 | [feature.delete() for feature in features] 90 | -------------------------------------------------------------------------------- /gcasc/instance_variables.py: -------------------------------------------------------------------------------- 1 | import re 2 | 3 | from .base import Configurer, Mode 4 | from .utils import diff, logger, objects 5 | from .utils.validators import ValidationResult 6 | 7 | logger = logger.get_logger("instance_variables") 8 | 9 | 10 | class InstanceVariablesConfigurer(Configurer): 11 | _NAME = "instance_variables" 12 | _SCHEMA = """ 13 | type: object 14 | propertyNames: 15 | pattern: "^[a-zA-Z0-9_]*$" 16 | maxLength: 255 17 | patternProperties: 18 | "^.*$": 19 | type: 20 | - object 21 | - string 22 | - boolean 23 | - integer 24 | maxLength: 10000 25 | required: 26 | - value 27 | properties: 28 | masked: 29 | type: boolean 30 | protected: 31 | type: boolean 32 | value: 33 | maxLength: 10000 34 | type: 35 | - string 36 | - boolean 37 | - integer 38 | variable_type: 39 | type: string 40 | enum: 41 | - file 42 | - env_var 43 | additionalProperties: false 44 | """ 45 | 46 | def __init__( 47 | self, gitlab, instance_variables, mode=Mode.APPLY 48 | ): # type: (gitlab.Gitlab, dict, Mode)->InstanceVariablesConfigurer 49 | super().__init__(gitlab, instance_variables, mode=mode) 50 | 51 | def __map_variable(self, k, v): 52 | return {**v, "key": k} if isinstance(v, dict) else {"value": v, "key": k} 53 | 54 | def configure(self): 55 | logger.info("Configuring Instance Variables") 56 | existing_variables = self.gitlab.variables.list() 57 | list_conf = [self.__map_variable(k, v) for k, v in self.config.items()] 58 | result = diff.diff_list(list_conf, existing_variables, "key") 59 | if result.has_changes(): 60 | self._remove(result.remove) 61 | self._create(result.create) 62 | self._update(result.update) 63 | else: 64 | logger.info("Nothing has changed") 65 | 66 | def _remove(self, variables): 67 | for var in variables: 68 | logger.info("Removing: %s", var.key) 69 | if self.mode == Mode.APPLY: 70 | var.delete() 71 | 72 | def _update(self, variables): 73 | for new_values, var in variables: 74 | logger.info("Updating: %s", var.key) 75 | objects.update_object(var, new_values) 76 | if self.mode == Mode.APPLY: 77 | var.save() 78 | 79 | def _create(self, variables): 80 | for var in variables: 81 | logger.info("Creating: %s", var.get("key")) 82 | if self.mode == Mode.APPLY: 83 | self.gitlab.variables.create(var) 84 | 85 | def _validate(self, errors): # type: (ValidationResult) -> () 86 | for key, variable in self.config.items(): 87 | is_dict = isinstance(variable, dict) 88 | if not is_dict: 89 | continue 90 | value = variable.get("value") if is_dict else variable 91 | if value is None: 92 | continue 93 | path = [key, "value"] 94 | if variable.get("masked"): 95 | if len(value) < 8: 96 | errors.add("must have at least 8 chars to be masked", path=path) 97 | if "\n" in value: 98 | errors.add("must be in a single line to be masked", path=path) 99 | # https://gitlab.com/gitlab-org/gitlab/-/blob/fd3a3a8f75f7bddc7c02dc9cf178986bc008ae60/app/models/concerns/ci/maskable.rb 100 | if not bool(re.match(r"\A[a-zA-Z0-9_+=/@:.~\-]+\Z", value)): 101 | errors.add( 102 | "must consist only of characters from Base64 alphabet plus '@', ':', '-'", 103 | path=path, 104 | ) 105 | 106 | return errors 107 | -------------------------------------------------------------------------------- /gcasc/license.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | 3 | from .base import Configurer, Mode 4 | from .utils import logger 5 | 6 | logger = logger.get_logger("License") 7 | 8 | 9 | class LicenseConfigurer(Configurer): 10 | _NAME = "license" 11 | _SCHEMA = """ 12 | type: object 13 | required: 14 | - starts_at 15 | - expires_at 16 | - plan 17 | - user_limit 18 | - data 19 | properties: 20 | starts_at: 21 | type: string 22 | format: date 23 | expires_at: 24 | type: string 25 | format: date 26 | plan: 27 | type: string 28 | enum: 29 | - starter 30 | - premium 31 | - ultimate 32 | user_limit: 33 | type: number 34 | minimum: 1 35 | data: 36 | type: string 37 | additionalProperties: false 38 | """ 39 | 40 | def __init__( 41 | self, gitlab, license, mode=Mode.APPLY 42 | ): # type: (gitlab.Gitlab, dict, Mode)->LicenseConfigurer 43 | super().__init__(gitlab, license, mode=mode) 44 | 45 | def configure(self): 46 | logger.info("Configuring GitLab licenses") 47 | current_license = self.gitlab.get_license() 48 | if ( 49 | not self.__check_if_same_license(current_license) 50 | and self.mode == Mode.APPLY 51 | ): 52 | current_license = self._update_license() 53 | logger.info( 54 | "Current license:\nplan: %s\nstarts_at: %s\nexpires_at: %s\nuser_limit: %s", 55 | self.__get_plan(current_license), 56 | self.__get_starts_at(current_license), 57 | self.__get_expires_at(current_license), 58 | self.__get_user_limit(current_license), 59 | ) 60 | return current_license 61 | 62 | def __get(self, field, license=None): # type: (str, dict) -> str 63 | license_config = license if license is not None else self.config 64 | return license_config.get(field) 65 | 66 | def __get_date(self, field, license=None): 67 | date = self.__get(field, license) 68 | return date.strftime("%Y-%m-%d") if isinstance(date, datetime.date) else date 69 | 70 | def __get_starts_at(self, license=None): # type: (dict) -> str 71 | return self.__get_date("starts_at", license) 72 | 73 | def __get_expires_at(self, license=None): # type: (dict) -> str 74 | return self.__get_date("expires_at", license) 75 | 76 | def __get_plan(self, license=None): # type: (dict) -> str 77 | return self.__get("plan", license) 78 | 79 | def __get_user_limit(self, license=None): # type: (dict) -> str 80 | return self.__get("user_limit", license) 81 | 82 | def __get_data(self, license=None): # type: (dict) -> str 83 | return self.__get("data", license) 84 | 85 | def __check_if_same_license(self, license): 86 | return ( 87 | self.__get_starts_at() == self.__get_starts_at(license) 88 | and self.__get_expires_at() == self.__get_expires_at(license) 89 | and self.__get_plan() == self.__get_plan(license) 90 | and self.__get_user_limit() == self.__get_user_limit(license) 91 | ) 92 | 93 | def _update_license(self): 94 | logger.info("Updating GitLab license...") 95 | return self.gitlab.set_license(self.__get_data()) 96 | -------------------------------------------------------------------------------- /gcasc/settings.py: -------------------------------------------------------------------------------- 1 | from .base import Mode, UpdateOnlyConfigurer 2 | 3 | 4 | class SettingsConfigurer(UpdateOnlyConfigurer): 5 | _NAME = "settings" 6 | 7 | def __init__(self, gitlab, settings, mode=Mode.APPLY): 8 | super().__init__("ApplicationSettings", gitlab, settings, mode=mode) 9 | 10 | def _load(self): 11 | return self.gitlab.settings.get() 12 | -------------------------------------------------------------------------------- /gcasc/utils/__init__.py: -------------------------------------------------------------------------------- 1 | import yaml 2 | 3 | from gcasc.utils.yaml_env import YamlEnvConstructor 4 | 5 | yaml.add_constructor( 6 | u"tag:yaml.org,2002:timestamp", lambda self, node: self.construct_scalar(node) 7 | ) 8 | YamlEnvConstructor.add_to_loader_class(loader_class=yaml.FullLoader) 9 | -------------------------------------------------------------------------------- /gcasc/utils/diff.py: -------------------------------------------------------------------------------- 1 | class DiffResult: 2 | def __init__(self, create, update, remove, unchanged): 3 | self.create = create 4 | self.update = update 5 | self.remove = remove 6 | self.unchanged = unchanged 7 | 8 | def has_changes(self): 9 | return len(self.create) > 0 or len(self.update) > 0 or len(self.remove) > 0 10 | 11 | 12 | def diff_list(list1, list2, keys=None): 13 | if isinstance(keys, str): 14 | keys = [keys] 15 | list1_dict = to_map(list1, keys) 16 | list2_dict = to_map(list2, keys) 17 | return __compare(list1_dict, list2_dict) 18 | 19 | 20 | def to_map(list, keys): 21 | if keys is None or len(keys) == 0: 22 | # all fields are keys, so no mapping 23 | return list 24 | key_function = ( 25 | (lambda obj: obj[keys[0]]) 26 | if len(keys) == 1 27 | else (lambda obj: create_tuple_key(obj, keys)) 28 | ) 29 | return {key_function(_unwrap_object(obj)): obj for obj in list} 30 | 31 | 32 | def _unwrap_object(obj): 33 | unwrapped = obj if isinstance(obj, dict) else obj.__dict__ 34 | # python-gitlab object props are wrapped in _attrs, so we should unwrap 35 | return unwrapped if "_attrs" not in unwrapped else unwrapped["_attrs"] 36 | 37 | 38 | def _filter_non_primitive(obj): 39 | return dict(filter(lambda tuple: _is_primitive(tuple[1]), obj.items())) 40 | 41 | 42 | def _is_primitive(obj): 43 | return type(obj) in (int, str, bool, list, dict, tuple) 44 | 45 | 46 | def create_tuple_key(obj, keys): 47 | return tuple(map(lambda key: obj.get(key), keys)) 48 | 49 | 50 | def __compare(obj1, obj2): 51 | create = [] 52 | update = [] 53 | unchanged = [] 54 | 55 | for key, value in obj1.items(): 56 | value2 = obj2.get(key) 57 | if value2 is None: 58 | create.append(value) 59 | else: 60 | if __compare_single(value, value2): 61 | unchanged.append(value) 62 | else: 63 | update.append((value, value2)) 64 | del obj2[key] 65 | remove = list(obj2.values()) 66 | 67 | return DiffResult(create, update, remove, unchanged) 68 | 69 | 70 | def __compare_single(obj1, obj2): 71 | obj2_unwrapped = _unwrap_object(obj2) 72 | for attr, value in obj1.items(): 73 | if obj2_unwrapped[attr] != value: 74 | return False 75 | return True 76 | -------------------------------------------------------------------------------- /gcasc/utils/logger.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | 4 | level_str = os.getenv("LOGGING_LEVEL", "INFO") 5 | level = logging.getLevelName(level_str) 6 | 7 | logging.basicConfig(format="[%(levelname)s] [%(name)s] %(message)s", level=level) 8 | 9 | 10 | def get_logger(name="gcasc"): 11 | return Logger(name) 12 | 13 | 14 | def _apply_masking(value, mask=False): 15 | return "***" if mask else value 16 | 17 | 18 | class Logger(object): 19 | def __init__(self, name): 20 | self.logger = logging.getLogger(name) 21 | 22 | def is_debug_enabled(self): 23 | return self.logger.isEnabledFor(logging.DEBUG) 24 | 25 | def log_update(self, field, old_value, new_value, masked=False): 26 | self.debug( 27 | "Changed value of %s from %s to %s", 28 | field, 29 | _apply_masking(old_value, masked), 30 | _apply_masking(new_value, masked), 31 | ) 32 | 33 | def log_create(self, value, field=None, masked=False): 34 | value_to_log = _apply_masking(value, masked) 35 | if field is None: 36 | self.debug("Created new object: %s", value_to_log) 37 | else: 38 | self.debug("Created new field %s with value %s", field, value_to_log) 39 | 40 | def log_delete(self, field): 41 | self.debug("Deleted field or object: %s", field) 42 | 43 | def debug(self, message, *args): 44 | self.logger.debug(message, *args) 45 | 46 | def error(self, message, *args): 47 | self.logger.error(message, *args) 48 | 49 | def warn(self, message, *args): 50 | self.logger.warning(message, *args) 51 | 52 | def info(self, message, *args): 53 | self.logger.info(message, *args) 54 | -------------------------------------------------------------------------------- /gcasc/utils/objects.py: -------------------------------------------------------------------------------- 1 | def update_object(current, new, prefix=""): # type: (dict, dict, str)->int 2 | for key, value in new.items(): 3 | if isinstance(value, dict): 4 | update_object(current, value, "{0}{1}_".format(prefix, key)) 5 | continue 6 | 7 | prefixed_key = "{0}{1}".format(prefix, key) 8 | if hasattr(current, prefixed_key): 9 | current_value = getattr(current, prefixed_key) 10 | if current_value != value: 11 | setattr(current, prefixed_key, value) 12 | -------------------------------------------------------------------------------- /gcasc/utils/os.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from . import strings 4 | 5 | 6 | def get_env_or_else(env, default=None): 7 | if isinstance(env, list): 8 | for val in env: 9 | value = os.getenv(val) 10 | if strings.is_not_blank(value): 11 | return value 12 | 13 | if isinstance(env, str): 14 | return os.getenv(env, default) 15 | 16 | return default 17 | -------------------------------------------------------------------------------- /gcasc/utils/strings.py: -------------------------------------------------------------------------------- 1 | def is_blank(string): 2 | # type: (str)-> bool 3 | return not (string and string.strip()) 4 | 5 | 6 | def is_not_blank(string): 7 | # type: (str)-> bool 8 | return bool(string and string.strip()) 9 | -------------------------------------------------------------------------------- /gcasc/utils/validators.py: -------------------------------------------------------------------------------- 1 | from jsonschema.exceptions import ValidationError as SchemaValidationError 2 | 3 | 4 | def rewrite_error_message(error): 5 | return error.message 6 | 7 | 8 | validator_type_message_generator = { 9 | "required": rewrite_error_message, 10 | "dependencies": rewrite_error_message, 11 | "type": lambda error: f"is not of type '{error.validator_value}'", 12 | "enum": lambda error: f"is not one of '{error.validator_value}'", 13 | "maxLength": lambda error: f"is too long (maximum {error.validator_value} characters expected)", 14 | "minLength": lambda error: f"is too short (minimum {error.validator_value} characters expected)", 15 | "format": lambda error: f"does not match '{error.validator_value}' format", 16 | "pattern": lambda error: f"does not match pattern '{error.validator_value}'", 17 | "maximum": lambda error: f"is greater than the maximum of {error.validator_value}", 18 | "minimum": lambda error: f"is less than the minimum of {error.validator_value}", 19 | "exclusiveMaximum": lambda error: f"is greater than or equal to the maximum of {error.validator_value}", 20 | "exclusiveMinimum": lambda error: f"is less than or equal to the minimum of {error.validator_value}", 21 | "multipleOf": lambda error: f"is not multiple of of {error.validator_value}", 22 | "minItems": lambda error: f"does not contain enough elements (minimum {error.validator_value} elements expected)", 23 | "maxItems": lambda error: f"contains too many elements (maximum {error.validator_value} elements expected)", 24 | "uniqueItems": lambda error: "has non-unique elements", 25 | "additionalProperties": lambda error: "contains unexpected properties", 26 | "const": rewrite_error_message, 27 | } 28 | 29 | 30 | def create_message(schema_error): # type: (SchemaValidationError) -> str 31 | return validator_type_message_generator.get( 32 | schema_error.validator, 33 | lambda error: f"did not pass {error.validator} ({error.validator_value}) validation", 34 | )(schema_error) 35 | 36 | 37 | class ValidationResult(object): 38 | def __init__(self, type=None, errors=None): 39 | self._errors = [] if errors is None else errors 40 | self.type = type 41 | 42 | def __create_full_message(self, message, path=None, *args): 43 | message = message.format(*args) if args else message 44 | full_path = self.__prepare_path(path) 45 | return f"{full_path} {message}" if len(full_path) > 0 else message 46 | 47 | def add(self, error, path=None, *args): # type: (str, list, *str) -> () 48 | self._errors.append( 49 | { 50 | "message": error, 51 | "full_message": self.__create_full_message(error, path, args), 52 | "path": path, 53 | } 54 | ) 55 | 56 | def __wrap(self, part): 57 | return f"[{part}]" if isinstance(part, int) else f"['{part}']" 58 | 59 | def __prepare_path(self, path): 60 | type = self.type if self.type is not None else "" 61 | if path is None: 62 | return type 63 | full_path = "" 64 | if isinstance(path, list): 65 | mapped = map(self.__wrap, path) 66 | full_path = "".join(mapped) 67 | elif isinstance(path, dict): 68 | pass 69 | return f"{type}{full_path}" 70 | 71 | def get(self): 72 | return self._errors 73 | 74 | def iterate(self, consumer): 75 | messages = map(lambda x: x["full_message"], self._errors) 76 | [consumer(message) for message in messages] 77 | 78 | def get_as_string(self): 79 | messages = map(lambda x: x["full_message"], self._errors) 80 | return "\n".join(messages) 81 | 82 | def has_errors(self): 83 | return len(self._errors) > 0 84 | 85 | def has_error(self, message=None, path=None): 86 | by_message = message is not None 87 | by_path = path is not None 88 | if by_message and by_path: 89 | return self.has_error_message_and_path(message, path) 90 | elif by_message and not by_path: 91 | return self.has_error_message(message) 92 | elif not by_message and by_path: 93 | return self.has_error_path(path) 94 | else: 95 | return False 96 | 97 | def has_error_message_and_path(self, message, path): 98 | path = path if isinstance(path, list) else [path] 99 | return ( 100 | next( 101 | ( 102 | e 103 | for e in self._errors 104 | if e["path"] == path and message in e["message"] 105 | ), 106 | None, 107 | ) 108 | is not None 109 | ) 110 | 111 | def has_error_path(self, path): 112 | path = path if isinstance(path, list) else [path] 113 | return next((e for e in self._errors if e["path"] == path), None) is not None 114 | 115 | def has_error_message(self, message): 116 | return ( 117 | next((e for e in self._errors if message in e["message"]), None) is not None 118 | ) 119 | -------------------------------------------------------------------------------- /gcasc/utils/yaml_env.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import os.path 4 | 5 | import yaml 6 | 7 | try: 8 | from yaml import FullLoader 9 | except ImportError: 10 | FullLoader = None 11 | 12 | __all__ = ["YamlEnvConstructor"] 13 | 14 | 15 | class YamlEnvConstructor: 16 | DEFAULT_TAG_NAME = "!env" 17 | 18 | def __call__(self, loader, node): 19 | args = [] 20 | if isinstance(node, yaml.nodes.ScalarNode): 21 | args = [loader.construct_scalar(node)] 22 | elif isinstance(node, yaml.nodes.SequenceNode): 23 | args = loader.construct_sequence(node) 24 | else: 25 | raise TypeError("Un-supported YAML node {!r}".format(node)) 26 | return YamlEnvConstructor.load(*args) 27 | 28 | @staticmethod 29 | def load(env): 30 | # type: (str)-> str 31 | splitted = env.split(":", 1) 32 | value = os.getenv(splitted[0], splitted[1] if len(splitted) == 2 else None) 33 | if value is None: 34 | raise RuntimeError( 35 | "Expected {0} environment variable, but value was not found in environment".format( 36 | env 37 | ) 38 | ) 39 | return value 40 | 41 | @classmethod 42 | def add_to_loader_class(cls, loader_class=None, tag=None, **kwargs): 43 | # type: (type(yaml.Loader), str, **str)-> YamlEnvConstructor 44 | 45 | if tag is None: 46 | tag = "" 47 | tag = tag.strip() 48 | if not tag: 49 | tag = cls.DEFAULT_TAG_NAME 50 | if not tag.startswith("!"): 51 | raise ValueError('`tag` argument should start with character "!"') 52 | instance = cls(**kwargs) 53 | if loader_class is None: 54 | if FullLoader: 55 | yaml.add_constructor(tag, instance, FullLoader) 56 | else: 57 | yaml.add_constructor(tag, instance) 58 | else: 59 | yaml.add_constructor(tag, instance, loader_class) 60 | return instance 61 | -------------------------------------------------------------------------------- /gcasc/utils/yaml_include.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # It is based on https://github.com/tanbro/pyyaml-include, but contains added handling of non yaml file types as strings 4 | 5 | """ 6 | Include YAML files within YAML 7 | """ 8 | 9 | import io 10 | import os.path 11 | import re 12 | from glob import iglob 13 | from sys import version_info 14 | 15 | import yaml 16 | 17 | try: 18 | from yaml import FullLoader 19 | except ImportError: 20 | FullLoader = None 21 | 22 | __all__ = ["YamlIncluderConstructor"] 23 | 24 | PYTHON_MAYOR_MINOR = "{0[0]}.{0[1]}".format(version_info) 25 | 26 | WILDCARDS_REGEX = re.compile(r"^.*(\*|\?|\[!?.+\]).*$") 27 | 28 | 29 | class YamlIncluderConstructor: 30 | """The `include constructor` for PyYAML Loaders 31 | 32 | Call :meth:`add_to_loader_class` or :meth:`yaml.Loader.add_constructor` to add it into loader. 33 | 34 | In YAML files, use ``!include`` to load other YAML files as below:: 35 | 36 | !include [dir/**/*.yml, true] 37 | 38 | or:: 39 | 40 | !include {pathname: dir/abc.yml, encoding: utf-8} 41 | 42 | """ 43 | 44 | DEFAULT_ENCODING = "utf-8" 45 | DEFAULT_TAG_NAME = "!include" 46 | 47 | def __init__(self, base_dir=None, encoding=None): 48 | # type:(str, str)->YamlIncluderConstructor 49 | """ 50 | :param str base_dir: Base directory where search including YAML files 51 | 52 | :default: ``None``: include YAML files from current working directory. 53 | 54 | :param str encoding: Encoding of the YAML files 55 | 56 | :default: ``None``: Not specified 57 | """ 58 | self._base_dir = base_dir 59 | self._encoding = encoding 60 | 61 | def __call__(self, loader, node): 62 | args = [] 63 | kwargs = {} 64 | if isinstance(node, yaml.nodes.ScalarNode): 65 | args = [loader.construct_scalar(node)] 66 | elif isinstance(node, yaml.nodes.SequenceNode): 67 | args = loader.construct_sequence(node) 68 | elif isinstance(node, yaml.nodes.MappingNode): 69 | kwargs = loader.construct_mapping(node) 70 | else: 71 | raise TypeError("Un-supported YAML node {!r}".format(node)) 72 | return self.load(loader, *args, **kwargs) 73 | 74 | @property 75 | def base_dir(self): # type: ()->str 76 | """Base directory where search including YAML files 77 | 78 | :rtype: str 79 | """ 80 | return self._base_dir 81 | 82 | @base_dir.setter 83 | def base_dir(self, value): # type: (str)->None 84 | self._base_dir = value 85 | 86 | @property 87 | def encoding(self): # type: ()->str 88 | """Encoding of the YAML files 89 | 90 | :rtype: str 91 | """ 92 | return self._encoding 93 | 94 | @encoding.setter 95 | def encoding(self, value): # type: (str)->None 96 | self._encoding = value 97 | 98 | @staticmethod 99 | def is_yaml(path): 100 | _, extension = os.path.splitext(path) 101 | return True if extension in [".yml", "yaml"] else False 102 | 103 | def load(self, loader, pathname, recursive=False, encoding=None): 104 | """Once add the constructor to PyYAML loader class, 105 | Loader will use this function to include other YAML fils 106 | on parsing ``"!include"`` tag 107 | 108 | :param loader: Instance of PyYAML's loader class 109 | :param str pathname: pathname can be either absolute (like /usr/src/Python-1.5/Makefile) or relative (like ../../Tools/*/*.gif), and can contain shell-style wildcards 110 | 111 | :param bool recursive: If recursive is true, the pattern ``"**"`` will match any files and zero or more directories and subdirectories. If the pattern is followed by an os.sep, only directories and subdirectories match. 112 | 113 | Note: 114 | Using the ``"**"`` pattern in large directory trees may consume an inordinate amount of time. 115 | 116 | :param str encoding: YAML file encoding 117 | 118 | :default: ``None``: Attribute :attr:`encoding` or constant :attr:`DEFAULT_ENCODING` will be used to open it 119 | 120 | :return: included YAML file, in Python data type 121 | 122 | .. warning:: It's called by :mod:`yaml`. Do NOT call it yourself. 123 | """ 124 | if not encoding: 125 | encoding = self._encoding or self.DEFAULT_ENCODING 126 | if self._base_dir: 127 | pathname = os.path.join(self._base_dir, pathname) 128 | if re.match(WILDCARDS_REGEX, pathname): 129 | result = [] 130 | if PYTHON_MAYOR_MINOR >= "3.5": 131 | iterator = iglob(pathname, recursive=recursive) 132 | else: 133 | iterator = iglob(pathname) 134 | for path in iterator: 135 | if os.path.isfile(path): 136 | with io.open( 137 | path, encoding=encoding 138 | ) as fp: # pylint:disable=invalid-name 139 | result.append(yaml.load(fp, type(loader))) 140 | return result 141 | with open(pathname, encoding=encoding) as fp: # pylint:disable=invalid-name 142 | if self.is_yaml(pathname): 143 | return yaml.load(fp, type(loader)) 144 | return fp.read() 145 | 146 | @classmethod 147 | def add_to_loader_class(cls, loader_class=None, tag=None, **kwargs): 148 | # type: (type(yaml.Loader), str, **str)-> YamlIncluderConstructor 149 | """ 150 | Create an instance of the constructor, and add it to the YAML `Loader` class 151 | 152 | :param loader_class: The `Loader` class add constructor to. 153 | 154 | .. attention:: This parameter **SHOULD** be a **class type**, **NOT** object. 155 | 156 | It's one of following: 157 | 158 | - :class:`yaml.BaseLoader` 159 | - :class:`yaml.UnSafeLoader` 160 | - :class:`yaml.SafeLoader` 161 | - :class:`yaml.Loader` 162 | - :class:`yaml.FullLoader` 163 | - :class:`yaml.CBaseLoader` 164 | - :class:`yaml.CUnSafeLoader` 165 | - :class:`yaml.CSafeLoader` 166 | - :class:`yaml.CLoader` 167 | - :class:`yaml.CFullLoader` 168 | 169 | :default: ``None``: 170 | 171 | - When :mod:`pyyaml` 3.*: Add to PyYAML's default `Loader` 172 | - When :mod:`pyyaml` 5.*: Add to `FullLoader` 173 | 174 | :type loader_class: type 175 | 176 | :param str tag: Tag's name of the include constructor. 177 | 178 | :default: ``""``: Use :attr:`DEFAULT_TAG_NAME` as tag name. 179 | 180 | :param kwargs: Arguments passed to construct function 181 | 182 | :return: New created object 183 | :rtype: YamlIncluderConstructor 184 | """ 185 | if tag is None: 186 | tag = "" 187 | tag = tag.strip() 188 | if not tag: 189 | tag = cls.DEFAULT_TAG_NAME 190 | if not tag.startswith("!"): 191 | raise ValueError('`tag` argument should start with character "!"') 192 | instance = cls(**kwargs) 193 | if loader_class is None: 194 | if FullLoader: 195 | yaml.add_constructor(tag, instance, FullLoader) 196 | else: 197 | yaml.add_constructor(tag, instance) 198 | else: 199 | yaml.add_constructor(tag, instance, loader_class) 200 | return instance 201 | -------------------------------------------------------------------------------- /renovate.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": [ 3 | "config:base" 4 | ] 5 | } 6 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | -r rtd-requirements.txt 2 | python-gitlab>=2.5.0 3 | PyYAML>=5.3.1 4 | deepdiff>=5.0.2 5 | jsonschema>=3.2.0 6 | rfc3987>=1.3.8 -------------------------------------------------------------------------------- /rtd-requirements.txt: -------------------------------------------------------------------------------- 1 | jinja2>=2.10.3 2 | sphinx>=2.2.1 3 | recommonmark>=0.6 4 | sphinx-rtd-theme>=0.4.3 5 | sphinx-markdown-tables>=0.0.10 -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | 4 | from setuptools import find_packages, setup 5 | 6 | 7 | def clean_requirements(requirements): 8 | return [req for req in requirements if not req.startswith("-r")] 9 | 10 | 11 | with open("README.md", "r") as readme_file: 12 | readme = readme_file.read() 13 | 14 | with open("requirements.txt", "r") as reqs_file: 15 | requirements = reqs_file.readlines() 16 | requirements = clean_requirements(requirements) 17 | 18 | with open("test-requirements.txt", "r") as test_reqs_file: 19 | test_requirements = test_reqs_file.readlines() 20 | test_requirements = clean_requirements(test_requirements) 21 | 22 | setup( 23 | name="gitlab-configuration-as-code", 24 | use_scm_version=True, 25 | setup_requires=["setuptools_scm"], 26 | description="Manage GitLab configuration as code", 27 | long_description_content_type="text/markdown", 28 | long_description=readme, 29 | author="Mateusz Filipowicz", 30 | author_email="mateusz.filipowicz@roche.com", 31 | license="Apache-2.0", 32 | url="https://github.com/Roche/gitlab-configuration-as-code", 33 | keywords=["gitlab", "configuration-as-code"], 34 | packages=find_packages(), 35 | install_requires=requirements, 36 | tests_require=test_requirements, 37 | entry_points={"console_scripts": ["gcasc = gcasc.bin.gcasc:main"]}, 38 | classifiers=[ 39 | "Development Status :: 4 - Beta", 40 | "Environment :: Console", 41 | "Intended Audience :: System Administrators", 42 | "License :: OSI Approved :: Apache Software License", 43 | "Natural Language :: English", 44 | "Operating System :: POSIX", 45 | "Operating System :: Microsoft :: Windows", 46 | "Programming Language :: Python", 47 | "Programming Language :: Python :: 3", 48 | "Programming Language :: Python :: 3.4", 49 | "Programming Language :: Python :: 3.5", 50 | "Programming Language :: Python :: 3.6", 51 | "Programming Language :: Python :: 3.7", 52 | "Programming Language :: Python :: 3.8", 53 | ], 54 | ) 55 | -------------------------------------------------------------------------------- /test-requirements.txt: -------------------------------------------------------------------------------- 1 | pytest>=5.2.4 2 | pytest-cov>=2.8.1 3 | 4 | # pin mock version due to some weird issue 5 | # https://github.com/Roche/gitlab-configuration-as-code/runs/1736343549?check_suite_focus=true 6 | mock==4.0.2 7 | flake8>=3.7.9 8 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Roche/gitlab-configuration-as-code/b8361ba4cf654774443b48e4da05cb12bb5dabb1/tests/__init__.py -------------------------------------------------------------------------------- /tests/appearance_test.py: -------------------------------------------------------------------------------- 1 | from unittest.mock import Mock 2 | 3 | import pytest 4 | 5 | from gcasc import AppearanceConfigurer 6 | 7 | from .helpers import read_yaml 8 | 9 | 10 | @pytest.fixture() 11 | def appearance_valid(): 12 | return read_yaml("appearance_valid.yml")["appearance"] 13 | 14 | 15 | def test_appearance_not_updated_because_unchanged(appearance_valid): 16 | # given 17 | gitlab = Mock() 18 | appearance = Mock() 19 | appearance.title = appearance_valid["title"] 20 | appearance.description = appearance_valid["description"] 21 | appearance.header_message = appearance_valid["header"]["message"] 22 | gitlab.appearance.get.return_value = appearance 23 | configurer = AppearanceConfigurer(gitlab, appearance_valid) 24 | 25 | # when 26 | configurer.configure() 27 | 28 | # then 29 | gitlab.appearance.get.assert_called_once() 30 | appearance.save.assert_not_called() 31 | 32 | 33 | def test_appearance_modified(appearance_valid): 34 | # given 35 | gitlab = Mock() 36 | appearance = Mock() 37 | appearance.title = appearance_valid["title"] 38 | appearance.description = "modified description" 39 | appearance.header_message = appearance_valid["header"]["message"] 40 | gitlab.appearance.get.return_value = appearance 41 | configurer = AppearanceConfigurer(gitlab, appearance_valid) 42 | 43 | # when 44 | saved = configurer.configure() 45 | 46 | # then 47 | gitlab.appearance.get.assert_called_once() 48 | appearance.save.assert_called_once() 49 | assert saved.description == appearance_valid["description"] 50 | -------------------------------------------------------------------------------- /tests/config_test.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from gcasc import GcascException, GitlabConfiguration 4 | 5 | 6 | def test_error_raised_when_configuration_none(): 7 | with pytest.raises(GcascException): 8 | GitlabConfiguration(None) 9 | 10 | 11 | def test_error_raised_when_configuration_not_a_dict(): 12 | with pytest.raises(GcascException): 13 | GitlabConfiguration("str") 14 | -------------------------------------------------------------------------------- /tests/data/appearance_valid.yml: -------------------------------------------------------------------------------- 1 | appearance: 2 | title: "GitLab Instance Title" 3 | description: "Instance description" 4 | header: 5 | message: "header message" -------------------------------------------------------------------------------- /tests/data/dummycert.crt: -------------------------------------------------------------------------------- 1 | this is dummy SSL certificate -------------------------------------------------------------------------------- /tests/data/dummykey.key: -------------------------------------------------------------------------------- 1 | this is dummy SSL key -------------------------------------------------------------------------------- /tests/data/features_invalid.yml: -------------------------------------------------------------------------------- 1 | features: 2 | - users: 3 | - test -------------------------------------------------------------------------------- /tests/data/features_valid.yml: -------------------------------------------------------------------------------- 1 | features: 2 | - name: sourcegraph 3 | value: false 4 | - name: import_export_object_storage 5 | value: true 6 | - name: ci_enable_live_trace 7 | value: true -------------------------------------------------------------------------------- /tests/data/features_valid_canaries.yml: -------------------------------------------------------------------------------- 1 | features: 2 | - name: sourcegraph 3 | value: true 4 | projects: 5 | - group1/project1 6 | - group2/project2 7 | users: 8 | - user1 9 | - user2 -------------------------------------------------------------------------------- /tests/data/features_valid_canary.yml: -------------------------------------------------------------------------------- 1 | features: 2 | - name: sourcegraph 3 | value: true 4 | projects: 5 | - mygroup/myproject 6 | users: 7 | - testuser 8 | groups: 9 | - testgroup -------------------------------------------------------------------------------- /tests/data/gitlab.yml: -------------------------------------------------------------------------------- 1 | settings: 2 | auto_devops_enabled: True 3 | license: 4 | starts_at: 2019-11-17 5 | expires_at: 2019-12-17 6 | plan: premium 7 | user_limit: 30 8 | data: some_license_data -------------------------------------------------------------------------------- /tests/data/gitlab_config_invalid.cfg: -------------------------------------------------------------------------------- 1 | [global] 2 | url = https://my.gitlab.com 3 | ssl_verify = true -------------------------------------------------------------------------------- /tests/data/gitlab_config_valid.cfg: -------------------------------------------------------------------------------- 1 | [global] 2 | url = https://my.gitlab.com 3 | ssl_verify = false 4 | timeout = 5 5 | private_token = my_token 6 | api_version = 4 7 | -------------------------------------------------------------------------------- /tests/data/instance_variables_invalid.yml: -------------------------------------------------------------------------------- 1 | instance_variables: 2 | inv@lid: n@me 3 | var1: 4 | var2: 5 | protected: true 6 | var3: 7 | value: val 8 | masked: true 9 | var4: 10 | value: val 11 | variable_type: some_type 12 | -------------------------------------------------------------------------------- /tests/data/instance_variables_valid.yml: -------------------------------------------------------------------------------- 1 | instance_variables: 2 | var1: val1 3 | var2: 4 | value: val2 5 | protected: true 6 | var3: 7 | value: abzcdDEF123_+=/@:.~- 8 | masked: true 9 | -------------------------------------------------------------------------------- /tests/data/license_invalid_1.yml: -------------------------------------------------------------------------------- 1 | license: 2 | starts_at: invalid_date 3 | expires_at: another_invalid_date 4 | plan: wrong_plan -------------------------------------------------------------------------------- /tests/data/license_valid.yml: -------------------------------------------------------------------------------- 1 | license: 2 | starts_at: 2019-11-17 3 | expires_at: 2019-12-17 4 | plan: premium 5 | user_limit: 30 6 | data: some_license_data -------------------------------------------------------------------------------- /tests/data/settings_valid.yml: -------------------------------------------------------------------------------- 1 | settings: 2 | auto_devops_enabled: True 3 | help_page: 4 | text: 'Custom text displayed on the help page.' 5 | support_url: 'Alternate support URL for help page and help dropdown.' 6 | polling_interval_multiplier: 1 7 | throttle: 8 | unauthenticated: 9 | enabled: true 10 | allowed_domains: 11 | - a 12 | - b 13 | -------------------------------------------------------------------------------- /tests/data/yaml_env.yml: -------------------------------------------------------------------------------- 1 | env1: !env ENV1 2 | env_list: 3 | - !env ENV1 4 | - someval 5 | - !env ENV2 -------------------------------------------------------------------------------- /tests/data/yaml_include.yml: -------------------------------------------------------------------------------- 1 | inc1: !include yaml_include_f1.yml 2 | inc2: 3 | - !include yaml_include_f2.yml 4 | - !include yaml_include_txt.md -------------------------------------------------------------------------------- /tests/data/yaml_include_f1.yml: -------------------------------------------------------------------------------- 1 | f1: v1 2 | f2: 3 | - f21: v21 4 | - v2 -------------------------------------------------------------------------------- /tests/data/yaml_include_f2.yml: -------------------------------------------------------------------------------- 1 | k1: v1 2 | k2: 3 | - k21: v21 4 | - v2 -------------------------------------------------------------------------------- /tests/data/yaml_include_txt.md: -------------------------------------------------------------------------------- 1 | # header1 2 | 3 | - some data 4 | - in order 5 | 6 | And free text 7 | -------------------------------------------------------------------------------- /tests/diff_test.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from gcasc.utils import diff 4 | 5 | KEY = "key" 6 | 7 | 8 | class MyObject: 9 | def __init__(self, key, value): 10 | self.key = key 11 | self.value = value 12 | 13 | def save(self): 14 | return self.key 15 | 16 | 17 | def _create_list(key, value, additional=None): 18 | obj = {KEY: key, "value": value} 19 | if not additional is None and isinstance(additional, dict): 20 | obj = {**obj, **additional} 21 | return [obj] 22 | 23 | 24 | def test_diff_created(): 25 | # given 26 | list1 = _create_list("a", "b") 27 | list2 = [] 28 | 29 | # when 30 | result = diff.diff_list(list1, list2, KEY) 31 | 32 | # then 33 | _assert_diff_result(result, create=1) 34 | assert result.create[0] == list1[0] 35 | 36 | 37 | @pytest.mark.parametrize( 38 | "value", 39 | [ 40 | ("b", "c"), # string 41 | (1, 2), # int 42 | (1.1, 2.2), # float 43 | (True, False), # bool 44 | ([1, 1], [1, 2]), # list(number) 45 | (["a", "a"], ["a", "a", "c"]), # list(string) 46 | (("a", 1), ("b", 1)), # tuple 47 | ({"x": "y"}, {"x": "z"}), # dict 48 | ({"x": {"y": 1}}, {"x": {"y": 2}}), # nested dict 49 | ], 50 | ) 51 | def test_diff_updated_primitive(value): 52 | # given 53 | list1 = _create_list("a", value[0]) 54 | list2 = _create_list("a", value[1]) 55 | 56 | # when 57 | result = diff.diff_list(list1, list2, KEY) 58 | 59 | # then 60 | _assert_diff_result(result, update=1) 61 | assert result.update[0] == (list1[0], list2[0]) 62 | 63 | 64 | def test_diff_removed(): 65 | # given 66 | list1 = [] 67 | list2 = _create_list("a", "b") 68 | 69 | # when 70 | result = diff.diff_list(list1, list2, KEY) 71 | 72 | # then 73 | _assert_diff_result(result, remove=1) 74 | assert result.remove[0] == (list2[0]) 75 | 76 | 77 | def test_diff_unchanged(): 78 | # given 79 | list1 = _create_list("a", "a", additional={"c": 1, "d": [1]}) 80 | list2 = _create_list("a", "a", additional={"c": 1, "d": [1]}) 81 | 82 | # when 83 | result = diff.diff_list(list1, list2, KEY) 84 | 85 | # then 86 | _assert_diff_result(result, unchanged=1) 87 | assert result.unchanged[0] == (list1[0]) 88 | 89 | 90 | @pytest.mark.parametrize( 91 | "list2", 92 | [ 93 | _create_list("a", "a", {"save": (lambda x: x)}), 94 | [MyObject("a", "a")], 95 | _create_list("a", "a", {"save": MyObject("a", "a")}), 96 | ], 97 | ) 98 | def test_diff_discards_complex_values(list2): 99 | # given 100 | list1 = _create_list("a", "a") 101 | 102 | # when 103 | result = diff.diff_list(list1, list2, KEY) 104 | 105 | # then 106 | _assert_diff_result(result, unchanged=1) 107 | assert result.unchanged[0] == (list1[0]) 108 | 109 | 110 | def test_diff_on_multiple_keys(): 111 | # given 112 | list1 = [ 113 | {KEY: "a", "key2": 1, "value": 1}, 114 | {KEY: "b", "key2": "a", "value": 1}, 115 | {KEY: "c", "key2": 1, "value": 1}, 116 | {KEY: "e", "key2": 2, "value": 1}, 117 | ] 118 | list2 = [ 119 | {KEY: "a", "key2": 1, "value": 1}, 120 | {KEY: "b", "key2": "a", "value": 2}, 121 | {KEY: "c", "key2": 2, "value": 1}, 122 | {KEY: "d", "key2": 2, "value": 1}, 123 | ] 124 | 125 | # when 126 | result = diff.diff_list(list1, list2, [KEY, "key2"]) 127 | 128 | # then 129 | _assert_diff_result(result, create=2, update=1, remove=2, unchanged=1) 130 | 131 | 132 | def _assert_diff_result(result, create=0, update=0, remove=0, unchanged=0): 133 | assert len(result.create) == create 134 | assert len(result.update) == update 135 | assert len(result.remove) == remove 136 | assert len(result.unchanged) == unchanged 137 | -------------------------------------------------------------------------------- /tests/features_test.py: -------------------------------------------------------------------------------- 1 | from unittest.mock import MagicMock, Mock 2 | 3 | import pytest 4 | from pytest import mark 5 | 6 | from gcasc import FeaturesConfigurer, Mode 7 | from gcasc.exceptions import ValidationException 8 | 9 | from .helpers import not_raises, read_yaml 10 | 11 | 12 | @pytest.fixture() 13 | def features_valid(): 14 | return read_yaml("features_valid.yml")["features"] 15 | 16 | 17 | @pytest.fixture() 18 | def features_invalid(): 19 | return read_yaml("features_invalid.yml")["features"] 20 | 21 | 22 | @pytest.fixture() 23 | def features_valid_canary(): 24 | return read_yaml("features_valid_canary.yml")["features"] 25 | 26 | 27 | @pytest.fixture() 28 | def features_valid_canaries(): 29 | return read_yaml("features_valid_canaries.yml")["features"] 30 | 31 | 32 | def __mock_gitlab(features=[]): 33 | features_manager = Mock() 34 | features_manager.list.return_value = features 35 | return Mock(features=features_manager) 36 | 37 | 38 | def test_features_configuration_valid(features_valid): 39 | # given 40 | configurer = FeaturesConfigurer(Mock(), features_valid, Mode.TEST_SKIP_VALIDATION) 41 | 42 | # when 43 | with (not_raises(ValidationException)): 44 | configurer.validate() 45 | 46 | 47 | def test_features_configuration_invalid(features_invalid): 48 | # given 49 | configurer = FeaturesConfigurer(Mock(), features_invalid, Mode.TEST_SKIP_VALIDATION) 50 | 51 | # when 52 | with pytest.raises(ValidationException) as error: 53 | configurer.validate() 54 | 55 | # then 56 | result = error.value.result 57 | assert len(result.get()) == 2 58 | assert result.has_error(message="name", path=0) 59 | assert result.has_error(message="value", path=0) 60 | 61 | 62 | def test_existing_features_removed_before_applying(): 63 | # given 64 | feature1 = Mock() 65 | feature2 = Mock() 66 | features = [feature1, feature2] 67 | gitlab = __mock_gitlab(features) 68 | configurer = FeaturesConfigurer(gitlab, []) 69 | 70 | # when 71 | configurer.configure() 72 | 73 | # then 74 | feature1.delete.assert_called_once() 75 | feature2.delete.assert_called_once() 76 | 77 | 78 | def test_canaries_configured_when_in_config(features_valid_canary): 79 | # given 80 | feature = features_valid_canary[0] 81 | name = feature["name"] 82 | value = feature["value"] 83 | user = feature["users"][0] 84 | group = feature["groups"][0] 85 | project = feature["projects"][0] 86 | gitlab = __mock_gitlab() 87 | configurer = FeaturesConfigurer(gitlab, features_valid_canary) 88 | 89 | # when 90 | configurer.configure() 91 | 92 | # then 93 | gitlab.features.set.assert_any_call(name, value, feature_group=None, user=user) 94 | 95 | gitlab.features.set.assert_any_call(name, value, feature_group=None, group=group) 96 | 97 | gitlab.features.set.assert_any_call( 98 | name, value, feature_group=None, project=project 99 | ) 100 | 101 | 102 | def test_multiple_canaries_are_configured(features_valid_canaries): 103 | # given 104 | gitlab = __mock_gitlab() 105 | configurer = FeaturesConfigurer(gitlab, features_valid_canaries) 106 | 107 | # when 108 | configurer.configure() 109 | 110 | # then 111 | assert gitlab.features.set.call_count == 4 112 | 113 | 114 | @mark.parametrize("mode", [Mode.TEST, Mode.TEST_SKIP_VALIDATION]) 115 | def test_configuration_not_applied_when_in_mode_other_than_not_apply( 116 | features_valid, mode 117 | ): 118 | # given 119 | gitlab = __mock_gitlab() 120 | configurer = FeaturesConfigurer(gitlab, features_valid, mode) 121 | 122 | # when 123 | configurer.configure() 124 | 125 | # then 126 | assert gitlab.features.set.call_count == 0 127 | -------------------------------------------------------------------------------- /tests/gcasc_test.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import pytest 4 | from mock import Mock, patch 5 | 6 | from gcasc import (ClientInitializationException, GitlabConfigurationAsCode, 7 | Mode) 8 | from gcasc.exceptions import GcascException 9 | from gitlab import GitlabError 10 | from tests import helpers 11 | 12 | GITLAB_CLIENT_CONFIG_FILE = ["GITLAB_CLIENT_CONFIG", "GITLAB_CLIENT_CONFIG_FILE"] 13 | GITLAB_CLIENT_CERTIFICATE = ["GITLAB_CLIENT_CERT", "GITLAB_CLIENT_CERTIFICATE"] 14 | GITLAB_CLIENT_KEY = "GITLAB_CLIENT_KEY" 15 | GITLAB_CLIENT_URL = "GITLAB_CLIENT_URL" 16 | GITLAB_CLIENT_API_VERSION = "GITLAB_CLIENT_API_VERSION" 17 | GITLAB_CLIENT_TOKEN = "GITLAB_CLIENT_TOKEN" 18 | GITLAB_CLIENT_SSL_VERIFY = "GITLAB_CLIENT_SSL_VERIFY" 19 | GITLAB_CONFIG_FILE = ["GITLAB_CONFIG_FILE", "GITLAB_CONFIG_PATH"] 20 | 21 | GITLAB_MODE = "GITLAB_MODE" 22 | 23 | GITLAB_CONFIG_FILE_DEFAULT_PATHS = [ 24 | "/etc/python-gitlab.cfg", 25 | "/etc/gitlab.cfg", 26 | "~/.python-gitlab.cfg", 27 | "~/.gitlab.cfg", 28 | ] 29 | 30 | 31 | def __mock_gitlab(gitlab_class_mock): 32 | gitlab = Mock() 33 | gitlab.version.return_value = ("test", "test") 34 | gitlab_class_mock.from_config.return_value = gitlab 35 | gitlab_class_mock.return_value = gitlab 36 | return gitlab 37 | 38 | 39 | @pytest.fixture(scope="class", autouse=True) 40 | def configure_shared_environment(request): 41 | os.environ["GITLAB_CONFIG_FILE"] = helpers.get_file_path("gitlab.yml") 42 | 43 | gitlab_patch = patch("gitlab.Gitlab") 44 | gitlab_mock_class = gitlab_patch.__enter__() 45 | __mock_gitlab(gitlab_mock_class) 46 | 47 | def unpatch(): 48 | gitlab_patch.__exit__() 49 | 50 | def clean_env(): 51 | for key, value in os.environ.items(): 52 | if key.startswith("GITLAB_"): 53 | del os.environ[key] 54 | 55 | request.addfinalizer(unpatch) 56 | request.addfinalizer(clean_env) 57 | 58 | 59 | @patch("gitlab.Gitlab") 60 | def test_gitlab_client_created_from_config_file(gitlab_class_mock): 61 | # given 62 | config_path = helpers.get_file_path("gitlab_config_valid.cfg") 63 | os.environ["GITLAB_CLIENT_CONFIG_FILE"] = config_path 64 | __mock_gitlab(gitlab_class_mock) 65 | 66 | # when 67 | GitlabConfigurationAsCode() 68 | 69 | # then 70 | gitlab_class_mock.assert_called_once_with( 71 | private_token="my_token", 72 | url="https://my.gitlab.com", 73 | ssl_verify=False, 74 | api_version="4", 75 | ) 76 | 77 | 78 | @patch("gitlab.Gitlab") 79 | def test_gitlab_client_created_from_environment(gitlab_class_mock): 80 | os.environ["GITLAB_CLIENT_TOKEN"] = "token" 81 | os.environ["GITLAB_CLIENT_URL"] = "url" 82 | os.environ["GITLAB_CLIENT_API_VERSION"] = "api_version" 83 | os.environ["GITLAB_CLIENT_SSL_VERIFY"] = "ssl_verify" 84 | __mock_gitlab(gitlab_class_mock) 85 | 86 | # when 87 | GitlabConfigurationAsCode() 88 | 89 | # then 90 | gitlab_class_mock.assert_called_once_with( 91 | private_token="token", 92 | url="url", 93 | ssl_verify="ssl_verify", 94 | api_version="api_version", 95 | ) 96 | 97 | 98 | @patch("gitlab.Gitlab") 99 | def test_gitlab_client_created_from_file_and_environment(gitlab_class_mock): 100 | # given 101 | config_path = helpers.get_file_path("gitlab_config_invalid.cfg") 102 | os.environ["GITLAB_CLIENT_CONFIG_FILE"] = config_path 103 | os.environ["GITLAB_CLIENT_TOKEN"] = "token" 104 | __mock_gitlab(gitlab_class_mock) 105 | 106 | # when 107 | GitlabConfigurationAsCode() 108 | 109 | # then 110 | gitlab_class_mock.assert_called_once_with( 111 | private_token="token", 112 | url="https://my.gitlab.com", 113 | ssl_verify=True, 114 | api_version="4", 115 | ) 116 | 117 | 118 | def test_gitlab_config_loaded_from_file(): 119 | # given 120 | os.environ["GITLAB_CLIENT_TOKEN"] = "some_token" 121 | 122 | # when 123 | gcasc = GitlabConfigurationAsCode() 124 | 125 | # then 126 | assert gcasc.config.config == helpers.read_yaml("gitlab.yml") 127 | 128 | 129 | def test_session_initialized_when_config_provided(): 130 | # given 131 | certificate = helpers.get_file_path("dummycert.crt") 132 | key = helpers.get_file_path("dummykey.key") 133 | os.environ["GITLAB_CLIENT_TOKEN"] = "token" 134 | os.environ["GITLAB_CLIENT_CERTIFICATE"] = certificate 135 | os.environ["GITLAB_CLIENT_KEY"] = key 136 | 137 | # when 138 | gcasc = GitlabConfigurationAsCode() 139 | 140 | # then 141 | assert gcasc.gitlab.session.cert == (certificate, key) 142 | 143 | 144 | def test_error_raised_when_unable_to_create_gitlab_client(): 145 | with pytest.raises(ClientInitializationException) as error: 146 | GitlabConfigurationAsCode() 147 | 148 | assert "config file" in error 149 | assert "environment variables" in error 150 | 151 | 152 | def test_apply_mode_is_used_when_not_provided(): 153 | # given 154 | os.environ["GITLAB_CLIENT_TOKEN"] = "token" 155 | 156 | # when 157 | gcasc = GitlabConfigurationAsCode() 158 | 159 | #then 160 | assert gcasc.mode == Mode.APPLY 161 | 162 | def test_non_zero_return_code_on_error(): 163 | # given 164 | os.environ["GITLAB_CLIENT_TOKEN"] = "token" 165 | 166 | with patch.object(GitlabConfigurationAsCode, '_configure') as gcasc_mock_method: 167 | # when 168 | gcasc_mock_method.side_effect = GcascException() 169 | 170 | # then 171 | with pytest.raises(SystemExit): 172 | GitlabConfigurationAsCode().configure() 173 | 174 | # when 175 | gcasc_mock_method.side_effect = GitlabError() 176 | 177 | # then 178 | with pytest.raises(SystemExit): 179 | GitlabConfigurationAsCode().configure() 180 | -------------------------------------------------------------------------------- /tests/helpers.py: -------------------------------------------------------------------------------- 1 | import os 2 | from contextlib import contextmanager 3 | 4 | import pytest 5 | import yaml 6 | 7 | try: 8 | from StringIO import StringIO 9 | except ImportError: 10 | from io import StringIO 11 | 12 | 13 | def __context(): 14 | dir = os.getcwd() 15 | if dir.endswith("tests"): 16 | return dir 17 | return "{0}/tests".format(dir) 18 | 19 | 20 | CONTEXT = __context() 21 | 22 | 23 | def get_file_path(file): 24 | return "{0}/data/{1}".format(CONTEXT, file) 25 | 26 | 27 | def read_file(file): 28 | with open(get_file_path(file)) as f: 29 | data = f.read() 30 | return data 31 | 32 | 33 | def read_yaml(file): 34 | with open(get_file_path(file)) as f: 35 | data = yaml.load(f, Loader=yaml.FullLoader) 36 | return data 37 | 38 | 39 | def read_yaml_from_string(str): 40 | return yaml.load(StringIO(str), Loader=yaml.FullLoader) 41 | 42 | 43 | @contextmanager 44 | def not_raises(exception): 45 | try: 46 | yield 47 | except exception: 48 | raise pytest.fail("DID RAISE {0}".format(exception)) 49 | -------------------------------------------------------------------------------- /tests/instance_variables_test.py: -------------------------------------------------------------------------------- 1 | from unittest.mock import Mock 2 | 3 | import pytest 4 | 5 | from gcasc import InstanceVariablesConfigurer, Mode 6 | from gcasc.exceptions import ValidationException 7 | 8 | from .helpers import read_yaml 9 | 10 | 11 | @pytest.fixture() 12 | def variables_valid(): 13 | return read_yaml("instance_variables_valid.yml")["instance_variables"] 14 | 15 | 16 | @pytest.fixture() 17 | def variables_invalid(): 18 | return read_yaml("instance_variables_invalid.yml")["instance_variables"] 19 | 20 | 21 | def test_variables_unchanged(variables_valid): 22 | # given 23 | gitlab = Mock() 24 | 25 | var1 = Mock() 26 | var1.key = "var1" 27 | var1.value = variables_valid["var1"] 28 | 29 | var2 = Mock() 30 | var2.key = "var2" 31 | var2.value = variables_valid["var2"]["value"] 32 | var2.protected = variables_valid["var2"]["protected"] 33 | 34 | var3 = Mock() 35 | var3.key = "var3" 36 | var3.value = variables_valid["var3"]["value"] 37 | var3.masked = variables_valid["var3"]["masked"] 38 | 39 | variables = [var1, var2, var3] 40 | 41 | gitlab.variables.list.return_value = variables 42 | configurer = InstanceVariablesConfigurer(gitlab, variables_valid) 43 | 44 | # when 45 | configurer.configure() 46 | 47 | # then 48 | gitlab.variables.list.assert_called_once() 49 | var1.save.assert_not_called() 50 | var2.save.assert_not_called() 51 | var3.save.assert_not_called() 52 | 53 | 54 | def test_variables_changed(variables_valid): 55 | # given 56 | gitlab = Mock() 57 | 58 | var1 = Mock() 59 | var1.key = "var1" 60 | var1.value = "changed value" 61 | 62 | var2 = Mock() 63 | var2.key = "var2" 64 | var2.value = variables_valid["var2"]["value"] 65 | var2.protected = False 66 | 67 | not_exist = Mock() 68 | not_exist.key = "not existent" 69 | not_exist.value = "not existent" 70 | 71 | variables = [var1, var2, not_exist] 72 | 73 | gitlab.variables.list.return_value = variables 74 | configurer = InstanceVariablesConfigurer(gitlab, variables_valid) 75 | 76 | # when 77 | configurer.configure() 78 | 79 | # then 80 | gitlab.variables.list.assert_called_once() 81 | var1.save.assert_called_once() 82 | var2.save.assert_called_once() 83 | not_exist.delete.assert_called_once() 84 | gitlab.variables.create.assert_called_once_with( 85 | {**variables_valid["var3"], "key": "var3"} 86 | ) 87 | 88 | 89 | def test_variables_invalid(variables_invalid): 90 | # given 91 | configurer = InstanceVariablesConfigurer( 92 | Mock(), variables_invalid, Mode.TEST_SKIP_VALIDATION 93 | ) 94 | 95 | # when 96 | with pytest.raises(ValidationException) as error: 97 | configurer.validate() 98 | 99 | # then 100 | result = error.value.result 101 | assert len(result.get()) == 5 102 | assert result.has_error(message="pattern", path="inv@lid") 103 | assert result.has_error(message="not of type", path="var1") 104 | assert result.has_error(message="value", path="var2") 105 | assert result.has_error(message="8 chars", path=["var3", "value"]) 106 | assert result.has_error(message="not one of", path=["var4", "variable_type"]) 107 | -------------------------------------------------------------------------------- /tests/license_test.py: -------------------------------------------------------------------------------- 1 | from unittest.mock import Mock 2 | 3 | import pytest 4 | 5 | from gcasc import LicenseConfigurer, Mode 6 | from gcasc.exceptions import ValidationException 7 | 8 | from .helpers import not_raises, read_yaml 9 | 10 | 11 | @pytest.fixture() 12 | def license_valid(): 13 | return read_yaml("license_valid.yml")["license"] 14 | 15 | 16 | @pytest.fixture() 17 | def license_invalid(): 18 | return read_yaml("license_invalid_1.yml")["license"] 19 | 20 | 21 | def test_license_configuration_valid(license_valid): 22 | # given 23 | configurer = LicenseConfigurer(Mock(), license_valid, Mode.TEST_SKIP_VALIDATION) 24 | 25 | # when 26 | with (not_raises(ValidationException)): 27 | configurer.validate() 28 | 29 | 30 | def test_license_configuration_invalid(license_invalid): 31 | # given 32 | configurer = LicenseConfigurer(Mock(), license_invalid, Mode.TEST_SKIP_VALIDATION) 33 | 34 | # when 35 | with pytest.raises(ValidationException) as error: 36 | configurer.validate() 37 | 38 | # then 39 | result = error.value.result 40 | assert len(result.get()) == 5 41 | assert result.has_error_message("user_limit") 42 | assert result.has_error_message("data") 43 | assert result.has_error_path("expires_at") 44 | assert result.has_error_path("plan") 45 | assert result.has_error_path("starts_at") 46 | 47 | 48 | def test_license_not_updated_because_same_exists(license_valid): 49 | # given 50 | gitlab = Mock() 51 | gitlab.get_license.return_value = { 52 | "starts_at": license_valid["starts_at"], 53 | "expires_at": license_valid["expires_at"], 54 | "plan": license_valid["plan"], 55 | "user_limit": license_valid["user_limit"], 56 | } 57 | configurer = LicenseConfigurer(gitlab, license_valid) 58 | 59 | # when 60 | configurer.configure() 61 | 62 | # then 63 | gitlab.get_license.assert_called_once() 64 | gitlab.set_license.assert_not_called() 65 | 66 | 67 | def test_license_updated(license_valid): 68 | # given 69 | gitlab = Mock() 70 | gitlab.get_license.return_value = { 71 | "starts_at": "1900-01-01", 72 | "expires_at": license_valid["expires_at"], 73 | "plan": license_valid["plan"], 74 | "user_limit": license_valid["user_limit"], 75 | } 76 | configurer = LicenseConfigurer(gitlab, license_valid) 77 | 78 | # when 79 | configurer.configure() 80 | 81 | # then 82 | gitlab.get_license.assert_called_once() 83 | gitlab.set_license.assert_called_once_with(license_valid["data"]) 84 | -------------------------------------------------------------------------------- /tests/settings_test.py: -------------------------------------------------------------------------------- 1 | from unittest.mock import Mock 2 | 3 | import pytest 4 | 5 | from gcasc import Mode, SettingsConfigurer 6 | 7 | from .helpers import read_yaml 8 | 9 | 10 | @pytest.fixture() 11 | def settings_valid(): 12 | return read_yaml("settings_valid.yml")["settings"] 13 | 14 | 15 | def test_settings_not_updated_because_unchanged(settings_valid): 16 | # given 17 | gitlab = Mock() 18 | settings = Mock() 19 | settings.auto_devops_enabled = settings_valid["auto_devops_enabled"] 20 | settings.help_page_text = settings_valid["help_page"]["text"] 21 | settings.help_page_support_url = settings_valid["help_page"]["support_url"] 22 | settings.polling_interval_multiplier = settings_valid["polling_interval_multiplier"] 23 | settings.throttle_unauthenticated_enabled = settings_valid["throttle"][ 24 | "unauthenticated" 25 | ]["enabled"] 26 | settings.allowed_domains = settings_valid["allowed_domains"] 27 | gitlab.settings.get.return_value = settings 28 | configurer = SettingsConfigurer(gitlab, settings_valid) 29 | 30 | # when 31 | configurer.configure() 32 | 33 | # then 34 | gitlab.settings.get.assert_called_once() 35 | settings.save.assert_not_called() 36 | 37 | 38 | def test_settings_modified(settings_valid): 39 | # given 40 | gitlab = Mock() 41 | settings = Mock() 42 | settings.auto_devops_enabled = settings_valid["auto_devops_enabled"] 43 | settings.help_page_text = "modified help page text" 44 | settings.help_page_support_url = settings_valid["help_page"]["support_url"] 45 | settings.polling_interval_multiplier = settings_valid["polling_interval_multiplier"] 46 | gitlab.settings.get.return_value = settings 47 | configurer = SettingsConfigurer(gitlab, settings_valid) 48 | 49 | # when 50 | saved = configurer.configure() 51 | 52 | # then 53 | gitlab.settings.get.assert_called_once() 54 | settings.save.assert_called_once() 55 | assert saved.help_page_text == settings_valid["help_page"]["text"] 56 | 57 | 58 | def test_invalid_configuration_options_are_skipped(settings_valid): 59 | # given 60 | gitlab = Mock() 61 | settings = Mock() 62 | settings.help_page_text = settings_valid["help_page"]["text"] 63 | gitlab.settings.get.return_value = settings 64 | configurer = SettingsConfigurer(gitlab, settings_valid) 65 | 66 | # when 67 | configurer.configure() 68 | 69 | # then 70 | gitlab.settings.get.assert_called_once() 71 | 72 | 73 | @pytest.mark.parametrize("mode", [Mode.TEST, Mode.TEST_SKIP_VALIDATION]) 74 | def test_no_changes_in_not_apply_mode(mode, settings_valid): 75 | # given 76 | text = "modified help page text" 77 | gitlab = Mock() 78 | settings = Mock() 79 | settings.help_page_text = text 80 | gitlab.settings.get.return_value = settings 81 | configurer = SettingsConfigurer(gitlab, settings_valid, mode) 82 | 83 | # when 84 | saved = configurer.configure() 85 | 86 | # then 87 | gitlab.settings.get.assert_called_once() 88 | settings.save.assert_not_called() 89 | assert saved.help_page_text == text 90 | -------------------------------------------------------------------------------- /tests/yaml_env_test.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import pytest 4 | import yaml 5 | 6 | from gcasc.utils.yaml_env import YamlEnvConstructor 7 | 8 | from .helpers import read_yaml 9 | 10 | YamlEnvConstructor.add_to_loader_class(loader_class=yaml.FullLoader) 11 | 12 | ENV1_VAL = "envvar1" 13 | ENV2_VAL = "envvar2\nmultiline\ttabbed" 14 | 15 | 16 | @pytest.fixture(autouse=True) 17 | def prepare_environment(): 18 | # before 19 | os.environ["ENV1"] = ENV1_VAL 20 | os.environ["ENV2"] = ENV2_VAL 21 | # test 22 | yield 23 | # after 24 | os.unsetenv("ENV1") 25 | os.unsetenv("ENV2") 26 | 27 | 28 | def test_environment_injected_into_yaml(): 29 | # given 30 | file = "yaml_env.yml" 31 | 32 | # when 33 | data = read_yaml(file) 34 | 35 | # then 36 | assert data["env1"] == ENV1_VAL 37 | assert data["env_list"] == [ENV1_VAL, "someval", ENV2_VAL] 38 | -------------------------------------------------------------------------------- /tests/yaml_include_test.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import pytest 4 | import yaml 5 | 6 | from gcasc.utils.yaml_include import YamlIncluderConstructor 7 | 8 | from .helpers import read_file, read_yaml 9 | 10 | YamlIncluderConstructor.add_to_loader_class( 11 | loader_class=yaml.FullLoader, 12 | base_dir=os.path.dirname(os.path.realpath(__file__)) + "/data", 13 | ) 14 | 15 | 16 | @pytest.fixture() 17 | def file1(): 18 | return read_yaml("yaml_include_f1.yml") 19 | 20 | 21 | @pytest.fixture() 22 | def file2(): 23 | return read_yaml("yaml_include_f2.yml") 24 | 25 | 26 | @pytest.fixture() 27 | def file_txt(): 28 | return read_file("yaml_include_txt.md") 29 | 30 | 31 | def test_files_included_into_yaml(file1, file2, file_txt): 32 | # given 33 | file = "yaml_include.yml" 34 | 35 | # when 36 | data = read_yaml(file) 37 | 38 | # then 39 | assert data["inc1"] == file1 40 | assert data["inc2"] == [file2, file_txt] 41 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | minversion = 1.6 3 | skipsdist = True 4 | envlist = py38,py37,py36,black,flake 5 | 6 | [testenv] 7 | setenv = VIRTUAL_ENV={envdir} 8 | whitelist_externals = true 9 | usedevelop = True 10 | install_command = pip install {opts} {packages} 11 | 12 | deps = -r {toxinidir}/requirements.txt 13 | -r {toxinidir}/test-requirements.txt 14 | commands = 15 | pytest \ 16 | -o log_cli=true \ 17 | --log-cli-level=DEBUG \ 18 | --cov-report xml \ 19 | --cov-report term \ 20 | --cov=gcasc \ 21 | --verbose \ 22 | --color=yes \ 23 | --junitxml=out_report.xml \ 24 | ./tests 25 | 26 | [testenv:flake] 27 | commands = 28 | flake8 {posargs} gcasc/ 29 | 30 | [testenv:black] 31 | basepython = python3 32 | deps = -r{toxinidir}/requirements.txt 33 | -r{toxinidir}/test-requirements.txt 34 | black 35 | commands = 36 | black {posargs} gcasc 37 | 38 | [testenv:venv] 39 | commands = {posargs} 40 | 41 | [testenv:docs] 42 | commands = python3 setup.py build_sphinx 43 | 44 | [flake8] 45 | exclude = .git,.venv,.tox,dist,doc,*egg,build, 46 | ignore = E501, W503 --------------------------------------------------------------------------------