├── .github └── ISSUE_TEMPLATE │ ├── bug_report.md │ ├── documentation-report.md │ └── feature_request.md ├── .gitignore ├── CONTRIBUTING.md ├── LICENSE.md ├── Makefile ├── NOTICE ├── README.md ├── docs ├── Makefile └── source │ ├── _templates │ └── layout.html │ ├── access.rst │ ├── authz.rst │ ├── authz1.png │ ├── autoscaling.rst │ ├── cli.rst │ ├── conf.py │ ├── contents.rst │ ├── development_guide.rst │ ├── development_setup.rst │ ├── faq.rst │ ├── importing.rst │ ├── index.rst │ ├── launch_questions.png │ ├── launch_questions.rst │ ├── login1.png │ ├── organization1.png │ ├── pipeline1.png │ ├── pipeline2.png │ ├── pipelines.rst │ ├── plugins.rst │ ├── projects.rst │ ├── projects1.png │ ├── resources.rst │ ├── scheduling.rst │ ├── security.rst │ ├── settings.rst │ ├── setup.rst │ ├── snippet1.png │ ├── stage1.png │ ├── tutorial.rst │ ├── upgrades.rst │ ├── variables.rst │ ├── webhooks.rst │ ├── worker_pool1.png │ ├── worker_pool2.png │ └── workers.rst ├── manage.py ├── requirements.txt ├── setup ├── 0_common.sh ├── 1_prepare.sh ├── 2_database.sh ├── 3_application.sh ├── 4_superuser.sh ├── 5_tutorial.sh └── 6_services.sh └── vespene ├── __init__.py ├── admin.py ├── common ├── __init__.py ├── logger.py ├── plugin_loader.py ├── secrets.py ├── templates.py └── variables.py ├── config ├── __init__.py ├── core.py ├── database.py ├── interface.py ├── plugins.py └── workers.py ├── jinja2 ├── dir_index.j2 ├── generic_edit.j2 ├── generic_list.j2 ├── generic_new.j2 ├── index.j2 ├── pipeline_map.j2 ├── project_start_prompt.j2 ├── registration │ └── login.html └── theme.j2 ├── management ├── __init__.py └── commands │ ├── __init__.py │ ├── autoscaler.py │ ├── cleanup.py │ ├── generate_secret.py │ ├── generate_supervisor.py │ ├── project_control.py │ ├── test_webhook.py │ ├── tutorial_setup.py │ └── worker.py ├── manager ├── __init__.py ├── jobkick.py ├── output.py ├── permissions.py ├── secrets.py └── webhooks.py ├── migrations ├── 0001_initial.py ├── 0002_auto_20181027_1530.py ├── 0003_auto_20181101_2224.py ├── 0004_auto_20181105_2006.py ├── 0005_auto_20181105_2042.py ├── 0006_auto_20181105_2044.py ├── 0007_project_recursive.py ├── 0008_auto_20181106_2233.py ├── 0009_remove_workerpool_sudo_password.py └── __init__.py ├── models ├── __init__.py ├── base.py ├── build.py ├── organization.py ├── pipeline.py ├── project.py ├── service_login.py ├── snippet.py ├── ssh_key.py ├── stage.py ├── variable_set.py ├── worker.py └── worker_pool.py ├── plugins ├── __init__.py ├── authorization │ ├── __init__.py │ ├── group_required.py │ └── ownership.py ├── autoscale_executor │ ├── __init__.py │ └── shell.py ├── autoscale_planner │ ├── __init__.py │ └── stock.py ├── isolation │ ├── __init__.py │ ├── basic_container.py │ └── sudo.py ├── organizations │ ├── __init__.py │ └── github.py ├── output │ ├── __init__.py │ └── timestamp.py ├── scm │ ├── __init__.py │ ├── git.py │ ├── none.py │ └── svn.py ├── secrets │ ├── __init__.py │ └── basic.py ├── triggers │ ├── __init__.py │ ├── command.py │ └── slack.py └── variables │ ├── __init__.py │ ├── common.py │ ├── pipelines.py │ └── snippets.py ├── settings.py ├── static ├── css │ ├── all.css │ ├── all.min.css │ ├── bootstrap.css │ ├── bootstrap.css.map │ ├── bootstrap.min.css │ ├── bootstrap.min.css.map │ ├── fontawesome.css │ ├── fontawesome.min.css │ ├── regular.css │ ├── regular.min.css │ ├── solid.css │ ├── solid.min.css │ └── vespene.css ├── js │ ├── all.js │ ├── all.min.js │ ├── bootstrap.js │ ├── bootstrap.min.js │ ├── fontawesome.js │ ├── fontawesome.min.js │ ├── jquery-3.3.1.min.js │ ├── regular.js │ ├── regular.min.js │ ├── solid.js │ ├── solid.min.js │ └── vespene.js ├── png │ └── vespene_logo.png └── webfonts │ ├── fa-brands-400.eot │ ├── fa-brands-400.svg │ ├── fa-brands-400.ttf │ ├── fa-brands-400.woff │ ├── fa-brands-400.woff2 │ ├── fa-regular-400.eot │ ├── fa-regular-400.svg │ ├── fa-regular-400.ttf │ ├── fa-regular-400.woff │ ├── fa-regular-400.woff2 │ ├── fa-solid-900.eot │ ├── fa-solid-900.svg │ ├── fa-solid-900.ttf │ ├── fa-solid-900.woff │ └── fa-solid-900.woff2 ├── version.py ├── views ├── __init__.py ├── build.py ├── fileserving.py ├── forms.py ├── group.py ├── organization.py ├── pipeline.py ├── project.py ├── service_login.py ├── snippet.py ├── ssh_key.py ├── stage.py ├── urls.py ├── user.py ├── variable_set.py ├── view_helpers.py └── worker_pool.py ├── workers ├── __init__.py ├── builder.py ├── commands.py ├── daemon.py ├── importer.py ├── isolation.py ├── pipelines.py ├── registration.py ├── scheduler.py ├── scm.py ├── ssh_agent.py └── triggers.py └── wsgi.py /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Submit a bug report 4 | 5 | --- 6 | 7 | **Bug description** 8 | 9 | **Steps to reproduce** 10 | 11 | **Expected behavior** 12 | 13 | **Actual behavior** 14 | 15 | **Operating system and version** 16 | 17 | **Any additional info** 18 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/documentation-report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Documentation report 3 | about: Submit a ticket about the documentation 4 | 5 | --- 6 | 7 | <-- FYI: If you would like to edit the docs yourself, the source lives in docs/source/*.rst in the repo. 8 | --> 9 | 10 | ** Explanation ** 11 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | 5 | --- 6 | 7 | Hi - 8 | 9 | We don't track feature requests in Github but would very much like to hear about your idea. 10 | 11 | Please stop by the forum at https://talk.msphere.io/c/ideas 12 | 13 | Thanks! 14 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | *.sqlite3 3 | *.pyc 4 | *.pyo 5 | *.pyd 6 | *.rdb 7 | *.mp4 8 | vespene/local_settings.py 9 | docs/build/ 10 | docs/source/vespene_logo.png 11 | __pycache__/ 12 | .svn/* 13 | /static/ 14 | /env/* 15 | *.swp 16 | *.mp4 17 | 18 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | Thank you for interest in Vespene! Pull requests are very welcome. 2 | 3 | Please read the official development guide [here](http://docs.vespene.io/development_guide.html) for some notes, tricks, and a bit about our community preferences. It will be expanded over time. 4 | 5 | As mentioned in the guide, be sure to join [http://talk.msphere.io](http://talk.msphere.io) if you haven't already. 6 | 7 | 8 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | requirements: 2 | pip install -r requirements.txt --trusted-host pypi.org --trusted-host files.pypi.org --trusted-host files.pythonhosted.org 3 | 4 | secrets: 5 | python manage.py generate_secret 6 | 7 | collectstatic: 8 | python manage.py collectstatic 9 | 10 | html: 11 | (rm -rf docs/build/html) 12 | (rm -rf docs/build/doctrees) 13 | cp vespene/static/png/vespene_logo.png docs/source/ 14 | (cd docs; make html) 15 | 16 | docs_publish: 17 | cp -a docs/build/html/* ../vespene-io.github.io/ 18 | 19 | indent_check: 20 | pep8 --select E111 vespene/ 21 | 22 | pyflakes: 23 | pyflakes vespene/ 24 | pyflakes vespene/views/*.py 25 | 26 | worker: 27 | ssh-agent env/bin/python manage.py worker general 28 | 29 | tutorial_setup: 30 | env/bin/python manage.py tutorial_setup 31 | 32 | clean: 33 | find . -name '*.pyc' | xargs rm -r 34 | find . -name '__pycache__' | xargs rm -rf 35 | 36 | migrations: 37 | PYTHONPATH=. env/bin/python manage.py makemigrations vespene 38 | 39 | migrate: 40 | PYTHONPATH=. env/bin/python manage.py migrate 41 | PYTHONPATH=. env/bin/python manage.py migrate vespene 42 | 43 | superuser: 44 | PYTHONPATH=. env/bin/python manage.py createsuperuser 45 | 46 | changepassword: 47 | PYTHONPATH=. env/bin/python manage.py changepassword 48 | 49 | uwsgi: 50 | uwsgi --http :8003 --wsgi-file vespene/wsgi.py -H env --plugins python3 --static-map /static=static 51 | 52 | supervisor_setup_example: 53 | PYTHONPATH=. env/bin/python manage.py supervisor_generate --executable /usr/bin/python --controller true --workers "tutorial-pool=2" 54 | 55 | run: 56 | PYTHONPATH=. env/bin/python manage.py runserver 57 | 58 | todo: 59 | grep TODO -rn vespene 60 | 61 | bug: 62 | grep BUG -rn vespene 63 | 64 | fixme: 65 | grep FIXME -rn vespene 66 | 67 | gource: 68 | gource -s .06 -1280x720 --auto-skip-seconds .1 --hide mouse,progress,filenames --key --multi-sampling --stop-at-end --file-idle-time 0 --max-files 0 --background-colour 000000 --font-size 22 --title "Vespene" --output-ppm-stream - --output-framerate 30 | avconv -y -r 30 -f image2pipe -vcodec ppm -i - -b 65536K movie.mp4 69 | 70 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | This is Vespene. Accept no substitutes. 2 | 3 | https://github.com/vespene-io/vespene 4 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Vespene 2 | ======= 3 | 4 | Vespene is a reimagined build system and automation console, focused on ease of use and advanced 5 | capabilities. 6 | 7 | Vespene was designed with extremely large micro-service deployments in mind, but it equally usable 8 | for all kinds of IT environments. 9 | 10 | While new, Vespene is growing quickly. All ideas for improvement are fair game! 11 | 12 | Features 13 | ======== 14 | 15 | * A horizontally-scalable, highly-available architecture built on Python, Django, and PostgreSQL 16 | * A well-organized, straight-forward user interface 17 | * Distributed workers 18 | * Declarative configuration via .vespene files (optional) 19 | * Built-in pipelines - no DSLs to learn, use easy .vespene YAML or set them up graphically 20 | * SSH-agent integration lending script access to encrypted SSH keys 21 | * A flexible variable system with Jinja2 templating and integration with any tool that can consume YAML/JSON 22 | * Webhooks and scheduled builds 23 | * Easy-to-configure access controls 24 | * Self-service automation panels for all types of users 25 | * Docker or sudo-based build isolation for security 26 | * Triggers to publish builds, run checks, or send messages to Slack 27 | * A plugin system where nearly everything is extensible - 8 types of plugins to date! 28 | * Easy administration, deployment, and upgrades 29 | 30 | Status 31 | ====== 32 | 33 | Currently Vespene is in "beta" status. 34 | 35 | You should feel comfortable running off the master branch today, and reliable 36 | database migrations are in place to enable easy upgrades. 37 | 38 | Our first release branch release will be in January of 2019, with new releases following approximately 39 | every 3 months. 40 | 41 | Documentation 42 | ============= 43 | 44 | For more on usage, capabilities, and setup, see [docs.vespene.io](http://docs.vespene.io). 45 | 46 | Requirements 47 | ============ 48 | 49 | The Vespene code requires one or more Linux or Unix environments that can run Python 3, 50 | and a PostgreSQL server, which we can help you install. 51 | 52 | Install automation is provided for the following platforms: 53 | 54 | * Ubuntu LTS distributions 55 | * CentOS 7 or RHEL 7 56 | * Arch Linux 57 | * openSUSE 58 | * OS X 59 | 60 | Automation for other install types are being added frequently. 61 | 62 | Installation Instructions 63 | ========================= 64 | 65 | The setup guide is [here](http://docs.vespene.io/setup.html). 66 | 67 | Forum & GitHub 68 | ============== 69 | 70 | If you have an idea or question, we'd encourage you to join the forum at [talk.msphere.io](http://talk.msphere.io). 71 | This is the best place to ask all questions about the project. 72 | 73 | To keep things organized, the issue tracker is just for bug tickets and pull requests. 74 | 75 | License 76 | ======= 77 | 78 | Vespene is Apache2 licensed. 79 | 80 | Author 81 | ====== 82 | 83 | Vespene is created and managed by Michael DeHaan . 84 | 85 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | SPHINXPROJ = Vespene 8 | SOURCEDIR = source 9 | BUILDDIR = build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 21 | -------------------------------------------------------------------------------- /docs/source/_templates/layout.html: -------------------------------------------------------------------------------- 1 | {% extends "!layout.html" %} 2 | {% block footer %} {{ super() }} 3 | 11 | 12 | 13 | 14 | 20 | 21 | {% endblock %} 22 | -------------------------------------------------------------------------------- /docs/source/access.rst: -------------------------------------------------------------------------------- 1 | 2 | .. image:: vespene_logo.png 3 | :alt: Vespene Logo 4 | :align: right 5 | 6 | .. _access: 7 | 8 | *************************** 9 | SSH Keys and Service Logins 10 | *************************** 11 | 12 | Vespene can store SSH private keys as well as service logins (such as GitHub username and passwords) to use during builds and checkouts that occur 13 | as part of builds. This means users using Vespene don't have to provide these credentials (or have direct access to them) and the system will use them on their behalf. 14 | 15 | .. _ssh: 16 | 17 | SSH Keys 18 | -------- 19 | 20 | Vespene can manage SSH keys in two ways. 21 | 22 | In the simplest case, when checking out repositories, such as git repos, Vespene can use SSH keys on your behalf during the checkout. 23 | Using dedicated SSH keys is often preferable to using usernames or passwords for these services. 24 | 25 | Further, when running projects, Vespene workers can use SSH keys to allow access to external systems. This is perhaps more interesting. For instance, 26 | a Vespene worker could use an SSH key to manage an external server or set of servers. 27 | 28 | Multiple SSH keys can be assigned to any project. They are entered in the "SSH Keys" view of Vespene, and then selected in the Project UI. 29 | Each key does require a private key upload, as well as an unlock password if the key is protected. 30 | 31 | The contents of the keys do not have to be shared with users who can access the Vespene UI, but they can still use them when launching the project. 32 | 33 | SSH keys uploaded *ARE* private keys, which are stored using Vespene encryption plugins in the database. 34 | 35 | Build isolation as described in :ref:`workers` is used to prevent the build scripts from accessing the database. As described in more detail in 36 | :ref:`security`, SSH keys given to Vespene should be deploy keys exclusively used by the Vespene system only, and frequently rotated. Key management 37 | may be modified in a future release. For improved security, keys given to Vespene should be unique for the purpose of use *by* Vespene to enable easy rotation. 38 | 39 | To use SSH keys it is required that workers are started wrapped with the 'ssh-agent' process, as described in :ref:`workers` and this is done automatically 40 | if you generate Vespene's supervisor config according to the :ref:`setup` instructions. 41 | 42 | Also note that there is no differentiation between SSH keys provided for access to a SCM or a machine, both are available for both purposes. If this is concerning, 43 | provide dedicated keys for specific purposes. 44 | 45 | .. _service_logins: 46 | 47 | Service Logins 48 | -------------- 49 | 50 | Service Logins are sets of usernames and passwords that can be used to access source control repositories. 51 | 52 | The system will not share the passwords used, but Service Logins are made available to multiple users. 53 | 54 | For source control systems that also work with SSH keys, like git, these can also be ignored in favor of :ref:`ssh`. 55 | 56 | At this time, Service Logins are *only* used during git checkouts and Subversion currently requires a publicly accessible repo. Updates to these 57 | behaviors are welcome contributions. 58 | 59 | These passwords are not yet marked by a particular service, for instance they can't be used for cloud API logins or something like that. This could 60 | also be implemented in the future. -------------------------------------------------------------------------------- /docs/source/authz1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vespene-io/_old_vespene/c0fbb217dfc5155bf974ab9e06fd43e539da63e0/docs/source/authz1.png -------------------------------------------------------------------------------- /docs/source/cli.rst: -------------------------------------------------------------------------------- 1 | .. image:: vespene_logo.png 2 | :alt: Vespene Logo 3 | :align: right 4 | 5 | .. _cli: 6 | 7 | CLI 8 | === 9 | 10 | Vespene has a few CLI commands available in addition to the web interface. 11 | 12 | These are "standard" "manage.py" Django management commands, which are easiest to execute by SSH'ing into a Vespene node. 13 | Django management commands provide an easy way to interface with the database without running through the whole web stack. 14 | 15 | Custom commands that ship with Vespene include: 16 | 17 | Job Execution 18 | ------------- 19 | 20 | Jobs may be manually started/stopped from the command line as follows: 21 | 22 | python manage.py job_control --project [--start|--stop] 23 | 24 | This may be extended to support project names and other options in the future. 25 | 26 | Tutorial setup 27 | -------------- 28 | 29 | As described in [the Tutorial](tutorial.html), this creates several objects for a basic demo: 30 | 31 | python manage.py tutorial_setup 32 | 33 | Cleanup 34 | ------- 35 | 36 | To clean up old build roots on a worker, older than a certain number of days, run this on each worker: 37 | 38 | python manage.py cleanup --remove-build-roots --days=30 39 | 40 | To clean up builds older than a certain number of days, run this on any node with database access: 41 | 42 | python manage.py cleanup --remove-builds --days=30 43 | 44 | The state of the build is not considered, only the age based on the time when the build was first queued. 45 | 46 | Stale queued builds (such as ones that were never started) are automatically cleaned up by the worker processes with no 47 | commands required to manage them. 48 | 49 | This would be a good command to consider adding to a crontab. 50 | 51 | Future 52 | ------ 53 | 54 | More commands will be added over time. If you have an idea for a useful command to contribute see :ref:`resources`. 55 | 56 | 57 | 58 | 59 | 60 | -------------------------------------------------------------------------------- /docs/source/contents.rst: -------------------------------------------------------------------------------- 1 | 2 | .. toctree:: 3 | :caption: Getting Started 4 | :maxdepth: 1 5 | 6 | About Vespene 7 | Setup Guide 8 | Tutorial 9 | 10 | .. toctree:: 11 | :caption: Fundamentals 12 | :maxdepth: 1 13 | 14 | Workers 15 | Projects 16 | Variables 17 | Access 18 | 19 | .. toctree:: 20 | :caption: Workflow 21 | :maxdepth: 1 22 | 23 | Imports (.vespene) 24 | Launch Questions 25 | Pipelines 26 | Scheduling 27 | Webhooks 28 | Autoscaling 29 | 30 | .. toctree:: 31 | :caption: Admin 32 | :maxdepth: 1 33 | 34 | Authorization 35 | CLI 36 | Plugins 37 | Security 38 | Settings 39 | Upgrades 40 | 41 | .. toctree:: 42 | :caption: Community 43 | :maxdepth: 1 44 | 45 | Resources 46 | Development Setup 47 | Development Guide 48 | FAQ / Troubleshooting 49 | 50 | -------------------------------------------------------------------------------- /docs/source/launch_questions.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vespene-io/_old_vespene/c0fbb217dfc5155bf974ab9e06fd43e539da63e0/docs/source/launch_questions.png -------------------------------------------------------------------------------- /docs/source/launch_questions.rst: -------------------------------------------------------------------------------- 1 | .. image:: vespene_logo.png 2 | :alt: Vespene Logo 3 | :align: right 4 | 5 | .. _launch_questions: 6 | 7 | **************** 8 | Launch Questions 9 | **************** 10 | 11 | Imagine if you want to have a project launch button that asks the user some questions, and then based on those 12 | answers, runs appropriately based on those answers. 13 | 14 | This is exactly what launch questions do. 15 | 16 | For instance, a generic deployment script could be written that prompts an ops engineer for the release tag, and can deploy any tag they 17 | type in. 18 | 19 | Similarly, a generic setup script for a new employee could ask for their username, and then provision lots of cloud resources for that user 20 | to create private development or test environments. 21 | 22 | Or also maybe imagine a project that reboots a region, or backs up a database, or anything. 23 | 24 | You could put very complex automation in the hands of someone who shoudn't (neccessarily) need to know how it works, and make it relatively 25 | fool-proof to run. 26 | 27 | Launch questions do just that. 28 | 29 | .. image:: launch_questions.png 30 | :alt: Example Launch Questions 31 | :align: left 32 | 33 | Setup 34 | ===== 35 | 36 | Launch questions can be configured on either a project under the "Variables" tab. 37 | 38 | The field takes JSON in list format, using a list of variable names and specifiers, as follows:: 39 | 40 | [ 41 | { "variable": "foo", "type": "multi", "choices" : [ "a", "b", "c" ] }, 42 | { "variable": "bar", "type": "radio", "choices" : [ "d", "e", "f" ] }, 43 | { "variable": "baz", "type": "text" } 44 | ] 45 | 46 | At this point, the validation on the question configuration is somewhat limited, so the best way to test the launch questions is by trying to 47 | start the project. 48 | 49 | The Question Format 50 | =================== 51 | 52 | Launch questions are specified by a JSON list of "question specifiers". 53 | 54 | Each question must have a type, or else it will assume to just use a "text" box, which is the default. 55 | 56 | The other two types at this time (more types may come later) are "radio" and "multi". 57 | Radio means to pick one item from a list, and multi allows picking more 58 | than one. 59 | 60 | The variable name here controls what variable name is to be provided to the templates, and the "prompt" controls 61 | what should be shown on the screen, as described below in "Usage" 62 | 63 | At this time there isn't much validation performed on the input values, but we're open to improvements. 64 | See :ref:`resources` and :ref:`development_guide` if you have ideas. 65 | 66 | Usage 67 | ===== 68 | 69 | When starting a project that has launch questions attached, instead of directly starting 70 | the project, a form will appear with the questions listed in order. 71 | 72 | Then, once started, the script for the given project (or all projects, if using a pipeline) can then 73 | access template variables:: 74 | 75 | #!/bin/bash 76 | ./step_one --foo="{{ foo }}" --bar="{{ bar }}" 77 | ./step_two --baz="{{ baz }}" 78 | 79 | As with standard builds, all of the variables made available to you are also available in 'vespene.json' in the 80 | build root, which is the current working directory of the build - so this is an easy way to pass it to 81 | any programming tool that can read JSON, or even YAML, because YAML is a subset of JSON. Many popular automation 82 | programs can read in variables this way. 83 | 84 | Limitations 85 | =========== 86 | 87 | When a project has launch questions attached, they are *NOT* prompted for when started by clicking on the 88 | pipeline, or if triggered by a webhook. The project will still run, the variables just won't be set. 89 | 90 | If you didn't define a default value for the variable elsewhere (like in the variables page 91 | for the project), this could result in a template error, which you can see in the build output. 92 | 93 | You could also code the Jinja2 template that is using the variable somewhat defensively to check to see if the 94 | value is set. There is also the Jinja2 filter called "| default". Refer to the Jinja2 documentation for 95 | more info. 96 | 97 | 98 | -------------------------------------------------------------------------------- /docs/source/login1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vespene-io/_old_vespene/c0fbb217dfc5155bf974ab9e06fd43e539da63e0/docs/source/login1.png -------------------------------------------------------------------------------- /docs/source/organization1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vespene-io/_old_vespene/c0fbb217dfc5155bf974ab9e06fd43e539da63e0/docs/source/organization1.png -------------------------------------------------------------------------------- /docs/source/pipeline1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vespene-io/_old_vespene/c0fbb217dfc5155bf974ab9e06fd43e539da63e0/docs/source/pipeline1.png -------------------------------------------------------------------------------- /docs/source/pipeline2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vespene-io/_old_vespene/c0fbb217dfc5155bf974ab9e06fd43e539da63e0/docs/source/pipeline2.png -------------------------------------------------------------------------------- /docs/source/projects.rst: -------------------------------------------------------------------------------- 1 | .. image:: vespene_logo.png 2 | :alt: Vespene Logo 3 | :align: right 4 | 5 | .. _projects: 6 | 7 | Projects 8 | ======== 9 | 10 | Projects are the cornerstone of all work in Vespene. 11 | 12 | A project in Vespene represents a source code repo with an associated build script, or maybe just a script without a source code repo at all. 13 | It could represent a push-button provisioning process just as well as building code. 14 | 15 | When you login to Vespene, the first view that comes up is a list of projects. 16 | 17 | .. image:: projects1.png 18 | :alt: Projects list 19 | :align: left 20 | 21 | Starting and Stopping 22 | --------------------- 23 | 24 | In the list view for a project, you can see the standard play/stop icons for starting and stopping a project build. 25 | 26 | Builds can also be started by :ref:`webhooks` - see that section for setup notes. 27 | 28 | Each project also maintains a list of build history. Cleaning up that history is described in :ref:`cli`. 29 | 30 | If you click on the edit button by a project there are many settings to configure. 31 | 32 | Creating A Project 33 | ------------------ 34 | 35 | A Project takes lots of parameters, and we'll cover some of those feature explanations in later chapters. 36 | 37 | Most importantly, a project needs a build script. 38 | 39 | The script *must* start with an interpreter line like "#!/bin/bash", so Vespene knows what to run. Otherwise any script can 40 | go in the script box. 41 | 42 | A project also needs a worker pool. This specifies what machines can run the particular build or script. 43 | 44 | Only superusers can create worker pools, and this is because they also require a running process to serve them. If no worker pools exist in Vespene, it will be impossible to create a project until one is created. Workers are not 45 | launched automatically, so admins will need to pay attention to :ref:`setup` and :ref:`workers` for more information on worker pool configuration. 46 | 47 | The script itself can use :ref:`variables` to simplify these scripts and reduce duplication between 48 | projects. 49 | 50 | In addition to the configuration of a repository address, Service Logins and SSH Keys (see :ref:`access`) can be attached to the repo to allow the project to access SCMs like GitHub or manage infrastructure using those keys. SSH key management is handled automatically by the workers using ssh-agent. 51 | 52 | The project can also be made a part of a continuous deployment pipeline. See :ref:`pipelines`. 53 | 54 | 55 | -------------------------------------------------------------------------------- /docs/source/projects1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vespene-io/_old_vespene/c0fbb217dfc5155bf974ab9e06fd43e539da63e0/docs/source/projects1.png -------------------------------------------------------------------------------- /docs/source/resources.rst: -------------------------------------------------------------------------------- 1 | .. image:: vespene_logo.png 2 | :alt: Vespene Logo 3 | :align: right 4 | 5 | .. _resources: 6 | 7 | Community Resources 8 | =================== 9 | 10 | If you are interested in getting more involved with Vespene, thanks, and here are some basic details! 11 | 12 | Contribution can include all types of things - including sharing ideas, testing, reporting a bug, giving a presentation, upgrading the documentation, and helping other users. Code is of course 13 | one of those things too! 14 | 15 | GitHub 16 | ------ 17 | 18 | If you would like to get involved with Vespene at a code level, "fork" the project at github.com/vespene/vespene. 19 | 20 | Please use GitHub for pull requests and filing bug tickets, but use `the forum `_ for questions and ideas and discussion. 21 | 22 | It is usually better to talk about features a little bit before coding them up, unless it is a simple plugin. 23 | 24 | This probably reduces frustration in most cases, and can prevent duplicate work efforts. We're happy to talk to people about what kinds of ways of implementing things would fit best in the codebase. 25 | 26 | 27 | See :ref:`development_setup` and :ref:`development_guide` for some introductory details. 28 | 29 | Please do not submit formatting corrections to the source, we don't take those - most everything else is great! 30 | 31 | Documentation Edits 32 | ------------------- 33 | 34 | If you want to work on the docs, the docs are actually in git too... just edit the 'rst' files in docs/source/ and give us a pull request. 35 | You can build the HTML versions of the docs yourself with "make html" from the root of the checkout. 36 | 37 | Twitter 38 | ------- 39 | 40 | Twitter is good for following developments. : `@vespene_io `_ 41 | 42 | Info/Announcement List 43 | ---------------------- 44 | 45 | For news and announcements, hit the email signup link at `http://vespene.io `_. 46 | (We will never give or sell your email addresses to anyone) 47 | 48 | Forum 49 | ----- 50 | 51 | The primary discussion forum for Vespene is `talk.msphere.io `_. 52 | 53 | 54 | 55 | 56 | 57 | 58 | -------------------------------------------------------------------------------- /docs/source/scheduling.rst: -------------------------------------------------------------------------------- 1 | .. image:: vespene_logo.png 2 | :alt: Vespene Logo 3 | :align: right 4 | 5 | .. _scheduling: 6 | 7 | Scheduling 8 | ---------- 9 | 10 | Vespene can be configured to run jobs on a periodic basis. 11 | 12 | To enable this feature, go to the "Scheduling" tab when editing a project. 13 | 14 | First, enable the "scheduling_enabled" checkbox, which is off by default. 15 | 16 | Afterwards, select what days the schedule will apply. 17 | 18 | Once the days are selected, you can now configure exactly what hour and minute combinations where a job will be queued up. 19 | 20 | This hour and minute combination setting is different for weekends and weekdays, allowing for optionally running a job 21 | less frequently on a weekend. 22 | 23 | It's not a fully flexible calendar system, because we just wanted to keep it fairly straightforward, but is possibly 24 | a good balance between something like cron and something more like repeating calendar invites. 25 | 26 | Example 27 | ======= 28 | 29 | To run a job every weekday at noon and 5pm Eastern time. 30 | Go into the "schedule" tab for a project. 31 | 32 | Check "schedule_enable" 33 | 34 | Check "monday", "tuesday", "wednesday", "thursday", and "friday" 35 | 36 | In the "weekday_start_hours" field enter "16, 19". 16 and 19 are the UTC hours for Noon and 5PM eastern. 37 | 38 | In the "weekday_start_minutes" field enter "0". This means to queue a job at exactly noon and 5pm. 39 | 40 | When Do Jobs Start 41 | ================== 42 | 43 | This system requires builders to build jobs, and it is quite possible that a builder will be busy and not 44 | start a job immediately. 45 | 46 | What will happen is that when a builder has time, it will consult the project configurations and queue up new 47 | jobs according to the schedule. 48 | 49 | When the workers reach for a new job to run, it may be one of the jobs that are queued up from the scheduler. 50 | If not, the job will run when it can. 51 | 52 | If you would like scheduled jobs to run promptly, consider allocating extra workers to the worker pool. 53 | 54 | The setting "schedule_threshold" in project settings is there as a safeguard against extraneous scheduling. For instance, 55 | if you want to run a project every 30 minutes, but a build occurred 10 minutes ago triggered by a pipeline that was ultimately triggered 56 | by a CI/CD webhook, you might not need to run a new build. The default schedule_threshold is 10 minutes. We don't recommend 57 | reducing it, but you may want to increase it to thirty (30) minutes or so to prevent infrastructure churn in some scenarios. 58 | 59 | Scheduler Considerations 60 | ======================== 61 | 62 | There needs to be at least one worker process running for jobs to get scheduled. 63 | 64 | If you would like 65 | to disable a scheduler during an outage window, you can uncheck the "schedule_enabled" checkbox and still leave all of the 66 | other scheduler settings saved in the database for when you re-enable it later. 67 | 68 | Scheduled jobs are kicked off by the backend and will intentionally ignore answering :ref:`launch_questions` that are set. 69 | If your project uses launch questions the template should have defaults, which are probably most easily set with the Jinja2 "| default" filter. 70 | 71 | Job scheduling logic, like all mogwai, may have some unexpected behavior around UTC midnight. Also, do not get the job scheduler wet. 72 | 73 | 74 | -------------------------------------------------------------------------------- /docs/source/snippet1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vespene-io/_old_vespene/c0fbb217dfc5155bf974ab9e06fd43e539da63e0/docs/source/snippet1.png -------------------------------------------------------------------------------- /docs/source/stage1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vespene-io/_old_vespene/c0fbb217dfc5155bf974ab9e06fd43e539da63e0/docs/source/stage1.png -------------------------------------------------------------------------------- /docs/source/upgrades.rst: -------------------------------------------------------------------------------- 1 | .. image:: vespene_logo.png 2 | :alt: Vespene Logo 3 | :align: right 4 | 5 | .. _upgrades: 6 | 7 | ******** 8 | Upgrades 9 | ******** 10 | 11 | Vespene upgrades always preserve data through use of built-in database migrations. 12 | 13 | The upgrade process for Vespene is as follows: 14 | 15 | Web nodes 16 | 17 | (1) Update the code on all nodes. 18 | 19 | Database 20 | ======== 21 | 22 | (2) Run "make migrate" on exactly one machine with database access 23 | to apply any database schema changes. 24 | 25 | Restart nodes 26 | ============= 27 | 28 | (3) Restart the webserver and worker processes 29 | 30 | If using systemd and the production setup:: 31 | 32 | # systemd restart vespene.service 33 | 34 | If not, just stop supervisord and restart it 35 | 36 | Verify 37 | ====== 38 | 39 | (4) Make sure everything is happy by logging into the interface 40 | and running a build or two. Make sure it is successful. 41 | 42 | 43 | 44 | -------------------------------------------------------------------------------- /docs/source/webhooks.rst: -------------------------------------------------------------------------------- 1 | .. image:: vespene_logo.png 2 | :alt: Vespene Logo 3 | :align: right 4 | 5 | .. _webhooks: 6 | 7 | ******** 8 | Webhooks 9 | ******** 10 | 11 | Using webhooks, a source control system can trigger Vespene builds automatically when new code is pushed. 12 | 13 | Configuration in the Source Control Product 14 | ------------------------------------------- 15 | 16 | Webhooks must be configured in your source control system, and you should consult their documentation for how to set this up. 17 | 18 | Webhooks should point to http://your-vespene-server.example.com/webhooks 19 | 20 | The URL you provide to something that generates webhooks will not change project by project. Vespene looks at the payload of the webhook to find out what repo the event is "about" 21 | and then matches up the repositories mentioned with your project. 22 | 23 | Here are two very important things to consider: 24 | 25 | * to avoid extra builds being triggered by extraneous events, only PUSH webhooks should be sent 26 | * all webhooks should be configured to send JSON 27 | 28 | Vespene has been tested with GitHub and support for GitLab *should* work at this time. For some source control systems, Vespene may need a VERY 29 | minor set of code additions to be able to support it. We welcome ALL of these additions and can work with you to get these added in ASAP if you send 30 | us a link to the webhook documentation. 31 | 32 | Configuration in Vespene 33 | ------------------------ 34 | 35 | For each project that should respond to a webhook with a new build, the enable webhook checkbox must be checked no the project. 36 | This checkbox is OFF by default, so no one can just configure up a webhook externally and start making Vespene fire off builds. 37 | 38 | Additionally, there is an optional field, 'webhook_token'. Normally this is blank, but this provides as a *BASIC* but generic 39 | extra safeguard. If supplied, the token must match the query string of the webhook. 40 | 41 | For instance, if the token is set to "badwolf", the URL for that project would have to be http://your-vespene-server.example.com/webhooks?token=badwolf -- you would use 42 | that URL instead of the generic webhook URL when setting the webhook up in git. If the webhook token doesn't match, the webhook will not fire. 43 | 44 | The webhook token can be unique per project. 45 | 46 | Network Relays 47 | -------------- 48 | 49 | Often a Vespene server on a private network will want to recieve webhooks from a hosted source control management service. 50 | 51 | There are commercial solutions for this (I have tested webhookrelay.com and it works great), though you could also write a POST forwarder yourself if you have any internet-facing web application. You would then only accept 52 | connections from your source control management system for the webhook URL, for example, the public IPs from github. 53 | 54 | Testing 55 | ------- 56 | 57 | Push to your repository and see if a build was triggered. 58 | 59 | See the logs/output from Vespene if needed for debug information. 60 | 61 | 62 | -------------------------------------------------------------------------------- /docs/source/worker_pool1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vespene-io/_old_vespene/c0fbb217dfc5155bf974ab9e06fd43e539da63e0/docs/source/worker_pool1.png -------------------------------------------------------------------------------- /docs/source/worker_pool2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vespene-io/_old_vespene/c0fbb217dfc5155bf974ab9e06fd43e539da63e0/docs/source/worker_pool2.png -------------------------------------------------------------------------------- /manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright 2018, Michael DeHaan LLC 3 | # License: Apache License Version 2.0 4 | 5 | import os 6 | import sys 7 | 8 | if __name__ == "__main__": 9 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "vespene.settings") 10 | try: 11 | from django.core.management import execute_from_command_line 12 | except ImportError as exc: 13 | raise ImportError( 14 | "Couldn't import Django. Are you sure it's installed and " 15 | "available on your PYTHONPATH environment variable? Did you " 16 | "forget to activate a virtual environment?" 17 | ) from exc 18 | execute_from_command_line(sys.argv) 19 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | Django==2.1.2 2 | psycopg2-binary==2.7.5 3 | requests==2.20.0 4 | jinja2==2.10 5 | django-crispy-forms==1.7.2 6 | sphinx-autobuild==0.7.1 7 | sphinx==1.7.6 8 | django-split-settings==0.3.0 9 | slackclient==1.2.1 10 | whitenoise==4.1 11 | sphinx_rtd_theme==0.4.1 12 | cryptography==2.3.1 13 | fernet==1.0.1 14 | gunicorn==19.9.0 15 | PyGithub==1.43.2 16 | PyYAML==3.13 17 | django-settings-export==1.2.1 18 | -------------------------------------------------------------------------------- /setup/0_common.sh: -------------------------------------------------------------------------------- 1 | # -- 2 | # /etc/vespene/settings.d/database.py info 3 | # -- 4 | 5 | # Database configuration 6 | # the local database is fine for an initial deployment 7 | # be sure to change the password! 8 | DBSERVER="127.0.0.1" 9 | DBPASS="vespene!" 10 | 11 | # -- 12 | # webserver info 13 | # -- 14 | 15 | # options to feed to gunicorn in /etc/vespene/supervisord.conf 16 | # if you change this to 0.0.0.0 to bind to all addresses be 17 | # sure to set up SSL. This will run as non-root so chose 18 | # a non-privileged port if adjusting the default. Proxying 19 | # with NGINX or Apache later is also an option. 20 | 21 | GUNICORN_OPTS="--bind 127.0.0.1:8000" 22 | 23 | # -- 24 | # /etc/vespene/settings.d/build.py info 25 | # -- 26 | 27 | BUILDROOT="/tmp/vespene" 28 | 29 | # --- 30 | # /etc/vespene/supervisord.conf info 31 | # --- 32 | 33 | # Should this machine be running any workers? 34 | # This is a space separated string of key=value pairs where the name is a 35 | # worker pool name (configured in the Vespene UI) and the value is the number of copies 36 | # of that worker to run. Increasing the number increases parallelism. 37 | 38 | # WORKER_CONFIG="general=2 tutorial-pool=1" 39 | WORKER_CONFIG="tutorial-pool=1" 40 | 41 | # how do you need to sudo to run the python management commands? 42 | APP_SUDO="sudo" 43 | 44 | # how do you need to sudo to run postgresql management commands? 45 | POST_SUDO="sudo -u postgres" 46 | 47 | # what filesystem user should own the postgresql data directory? 48 | DB_USER="postgres" 49 | 50 | # what user should be running Vespene itself? 51 | APP_USER="vespene" 52 | 53 | # rough OS detection for now; patches accepted! 54 | DISTRO="?" 55 | if [[ "$OSTYPE" == "linux-gnu" ]]; then 56 | if [ -f /etc/fedora-release ]; then 57 | echo "detected Fedora" 58 | DISTRO="fedora" 59 | PIP="/usr/bin/pip3" 60 | PYTHON="/usr/bin/python3" 61 | elif [ -f /etc/redhat-release ]; then 62 | echo "detected RHEL/CentOS" 63 | DISTRO="redhat" 64 | PIP="/usr/local/bin/pip3.6" 65 | PYTHON="/usr/bin/python3.6" 66 | elif [[ -f /etc/system-release && "$(cat /etc/system-release)" =~ "Amazon Linux release 2" ]]; then 67 | echo "detected Amazon Linux 2" 68 | DISTRO="amazon2" 69 | PIP="/usr/local/bin/pip3.6" 70 | PYTHON="/usr/bin/python3.6" 71 | elif [[ -f /etc/system-release && "$(cat /etc/system-release)" =~ "Amazon Linux AMI release" ]]; then 72 | echo "detected Amazon Linux" 73 | PIP="/usr/bin/pip-3.6" 74 | PYTHON="/usr/bin/python3.6" 75 | DISTRO="amazon" 76 | elif [ -f /usr/bin/zypper ]; then 77 | echo "detected openSUSE" 78 | DISTRO="opensuse" 79 | PYTHON="/usr/bin/python3" 80 | PIP="/usr/bin/pip3" 81 | elif [ -f /usr/bin/apt ]; then 82 | OS=$(lsb_release -si) 83 | if [ "$OS" == "Debian" ]; then 84 | echo "detected Debian" 85 | DISTRO="debian" 86 | PYTHON="/usr/bin/python3" 87 | PIP="/usr/bin/pip3" 88 | elif [ "$OS" == "Ubuntu" ]; then 89 | echo "detected Ubuntu" 90 | DISTRO="ubuntu" 91 | PYTHON="/usr/bin/python3" 92 | PIP="/usr/bin/pip3" 93 | fi 94 | elif [ -f /usr/bin/pacman ]; then 95 | echo "detected Arch Linux" 96 | DISTRO="archlinux" 97 | PYTHON="/usr/bin/python" 98 | PIP="/usr/bin/pip" 99 | fi 100 | elif [[ "$OSTYPE" == "linux-musl" ]]; then 101 | if [ -f /etc/alpine-release ]; then 102 | echo "detected Alpine Linux" 103 | DISTRO="alpine" 104 | PYTHON="/usr/bin/python3" 105 | PIP="/usr/bin/pip3" 106 | fi 107 | elif [ -f /usr/local/bin/brew ]; then 108 | echo "detected MacOS" 109 | DISTRO="MacOS" 110 | PYTHON="/usr/local/bin/python3" 111 | PIP="/usr/local/bin/pip3" 112 | APP_SUDO="" 113 | POST_SUDO="" 114 | DB_USER=`whoami` 115 | APP_USER=`whoami` 116 | else 117 | echo "this OS may work with Vespene but we don't have setup automation for this just yet" 118 | DISTRO="?" 119 | fi 120 | 121 | me=`whoami` 122 | -------------------------------------------------------------------------------- /setup/1_prepare.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | source ./0_common.sh 4 | 5 | echo "PIP=$PIP" 6 | 7 | #--- 8 | 9 | echo "Installing core packages..." 10 | if [ "$DISTRO" == "redhat" ]; then 11 | # this just covers CentOS for now 12 | sudo yum -y install centos-release-scl 13 | # on Red Hat 14 | # sudo yum-config-manager --enable rhel-server-rhscl-7-rpms ? 15 | sudo yum -y install epel-release 16 | sudo yum -y install gcc python36 python36-devel supervisor rh-postgresql10 openldap-devel 17 | sudo python36 -m ensurepip 18 | sudo tee /etc/profile.d/enable_pg10.sh >/dev/null << END_OF_PG10 19 | #!/bin/bash 20 | source scl_source enable rh-postgresql10 21 | END_OF_PG10 22 | sudo chmod +x /etc/profile.d/enable_pg10.sh 23 | elif [ "$DISTRO" == "fedora" ]; then 24 | sudo yum -y install gcc python3 python3-devel supervisor postgresql openldap-devel 25 | python3 -m ensurepip 26 | elif [ "$DISTRO" == "amazon2" ]; then 27 | sudo amazon-linux-extras install -y python3 postgresql10 28 | sudo yum -y install python2-pip 29 | sudo python -m pip install supervisor 30 | elif [[ "$DISTRO" == "amazon" ]]; then 31 | sudo yum -y install https://download.postgresql.org/pub/repos/yum/10/redhat/rhel-6-x86_64/pgdg-redhat10-10-2.noarch.rpm 32 | sudo sed -ie 's/rhel-\$releasever-\$basearch/rhel-6-x86_64/g' /etc/yum.repos.d/pgdg-10-redhat.repo 33 | sudo yum -y install python36 postgresql10 34 | sudo python -m pip install supervisor 35 | elif [ "$DISTRO" == "opensuse" ]; then 36 | zypper refresh 37 | zypper install -y gcc python python-pip python3 python3-pip python3-setuptools postgresql 38 | python2.7 -m pip install supervisor 39 | elif [ "$DISTRO" == "ubuntu" ]; then 40 | sudo apt-add-repository universe 41 | sudo apt-get update 42 | sudo apt-get install -y gcc libssl-dev postgresql-client python3 python3-pip python3-setuptools supervisor 43 | elif [ "$DISTRO" == "debian" ]; then 44 | sudo apt-get update 45 | sudo apt-get install -y gcc libssl-dev postgresql-client postgresql-server-dev-all python3-dev python3 python3-pip python3-setuptools supervisor musl-dev libffi-dev 46 | elif [ "$DISTRO" == "archlinux" ]; then 47 | sudo pacman --noconfirm -Sy python python-pip python-setuptools postgresql supervisor sudo 48 | elif [ "$DISTRO" == "alpine" ]; then 49 | sudo apk add musl-dev build-base python3-dev libffi-dev postgresql postgresql-dev supervisor 50 | elif [ "$DISTRO" == "MacOS" ]; then 51 | brew install python@3 postgresql supervisor 52 | fi 53 | 54 | #--- 55 | 56 | if [ "$DISTRO" != "MacOS" ]; then 57 | if [ "$DISTRO" != "alpine" ]; then 58 | sudo useradd vespene 59 | else 60 | sudo adduser -D vespene 61 | fi 62 | fi 63 | 64 | #--- 65 | 66 | echo "Setting up directories..." 67 | 68 | sudo mkdir -p /opt/vespene 69 | sudo mkdir -p /var/spool/vespene 70 | sudo mkdir -p /etc/vespene/settings.d/ 71 | sudo mkdir -p /var/log/vespene/ 72 | 73 | #--- 74 | 75 | echo "Cloning the project into /opt/vespene..." 76 | sudo rm -rf /opt/vespene/* 77 | sudo cp -a ../* /opt/vespene 78 | 79 | #--- 80 | 81 | echo "APP_USER=$APP_USER" 82 | sudo chown -R $APP_USER /opt/vespene 83 | sudo chown -R $APP_USER /var/spool/vespene 84 | sudo chown -R $APP_USER /etc/vespene/settings.d/ 85 | sudo chown -R $APP_USER /var/log/vespene 86 | 87 | #--- 88 | 89 | echo "Installing python packages..." 90 | CMD="sudo $PYTHON -m pip install -r ../requirements.txt --trusted-host pypi.org --trusted-host files.pypi.org --trusted-host files.pythonhosted.org" 91 | echo $CMD 92 | $CMD 93 | -------------------------------------------------------------------------------- /setup/3_application.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # --------------------------------------------------------------------------- 4 | # 3_application.sh: this step sets up Vespene configurations in /etc/vespene/settings.d, overriding 5 | # some defaults. It does not configure plugins, which are described in the online 6 | # documentation, so you'll get the default plugin configuration to start. 7 | # --------------------------------------------------------------------------- 8 | 9 | # this step also makes sure database tables are present on the database server, which 10 | # is set up by the previous script. 11 | 12 | # load common settings 13 | source ./0_common.sh 14 | 15 | #--- 16 | 17 | cd /opt/vespene 18 | echo $PYTHON 19 | sudo $PYTHON manage.py generate_secret 20 | 21 | #--- 22 | 23 | # application database config 24 | sudo tee /etc/vespene/settings.d/database.py >/dev/null </dev/null << END_OF_WORKERS 41 | BUILD_ROOT="${BUILDROOT}" 42 | # FILESERVING_ENABLED = True 43 | # FILESERVING_PORT = 8000 44 | # FILESERVING_HOSTNAME = "this-server.example.com" 45 | END_OF_WORKERS 46 | 47 | #--- 48 | 49 | # ui settings 50 | sudo tee /etc/vespene/settings.d/interface.py >/dev/null << END_OF_INTERFACE 51 | BUILDROOT_WEB_LINK="${BUILDROOT_WEB_LINK}" 52 | END_OF_INTERFACE 53 | 54 | #--- 55 | 56 | # authentication settings 57 | sudo tee /etc/vespene/settings.d/authentication.py >/dev/null << END_OF_AUTHENTICATION 58 | # Vespene uses the standard Django authentication system 59 | # the default authentication scheme uses the local database. to use LDAP: 60 | # 61 | # pip install python-ldap 62 | # pip install django-auth-ldap 63 | # 64 | # and uncomment these lines: 65 | # 66 | # import ldap 67 | # from django_auth_ldap.config import LDAPSearch, GroupOfNamesType 68 | 69 | # then configure LDAP as follows: 70 | # see example: https://django-auth-ldap.readthedocs.io/en/latest/example.html 71 | # 72 | # to enable LDAP authentication, uncomment django_auth_ldap.backend.LDAPBackend from AUTHENTICATION_BACKENDS. 73 | # to use LDAP exclusively, comment out django.contrib.auth.backends.ModelBackend 74 | # 75 | # See also http://docs.vespene.io/authz.html 76 | 77 | AUTHENTICATION_BACKENDS = ( 78 | #'django_auth_ldap.backend.LDAPBackend', 79 | 'django.contrib.auth.backends.ModelBackend', 80 | ) 81 | END_OF_AUTHENTICATION 82 | 83 | #--- 84 | 85 | # ensure app user can read all of this 86 | echo sudo chown -R $APP_USER /etc/vespene 87 | 88 | #--- 89 | 90 | # apply database tables 91 | # this only has to be run once but won't hurt anything by doing 92 | # it more than once. You also need this step during upgrades. 93 | cd /opt/vespene 94 | $APP_SUDO $PYTHON manage.py migrate 95 | -------------------------------------------------------------------------------- /setup/4_superuser.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # --------------------------------------------------------------------------- 4 | # 4_superuser.sh: prompt the user to create an admin account. Run this on any Vespene 5 | # machine with database access. 6 | # --------------------------------------------------------------------------- 7 | 8 | # load common settings 9 | source ./0_common.sh 10 | 11 | # run the django management command (this is interactive) 12 | cd /opt/vespene 13 | 14 | #--- 15 | 16 | echo $APP_SUDO 17 | $APP_SUDO $PYTHON manage.py createsuperuser 18 | -------------------------------------------------------------------------------- /setup/5_tutorial.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # --------------------------------------------------------------------------- 4 | # 5_tutorial.sh: sets up basic objects used in the tutorial, so Vespene isn't "blank" when you first log in. 5 | # these objects can be deleted later. 6 | # --------------------------------------------------------------------------- 7 | 8 | # this also creates a worker pool named "tutorial-pool" so one of our configured background 9 | # processes has something to do. 10 | 11 | # load common settings 12 | source ./0_common.sh 13 | 14 | #--- 15 | 16 | # run the custom management command that creates the objects 17 | cd /opt/vespene 18 | $APP_SUDO $PYTHON manage.py tutorial_setup 19 | 20 | 21 | -------------------------------------------------------------------------------- /setup/6_services.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # --------------------------------------------------------------------------- 4 | # 6_services.sh: sets up supervisor to start the various worker processes, if any are configured. The tutorial example typically 5 | # sets up a worker called "tutorial-pool", though you can have any number of them. To change these later, edit 6 | # the supervisor configuration in /etc/vespene/ and restart the vespene systemd service. 7 | 8 | # workers are coded so that if they don't have an object in the Vespene database yet those processes will sleep and wake 9 | # up every minute until they do. So if you created some named workers here you still need to create those names in the 10 | # Vespene UI before they can start doing any work. 11 | # --------------------------------------------------------------------------- 12 | 13 | # load common config settings 14 | 15 | source ./0_common.sh 16 | 17 | # --- 18 | 19 | # generate the supervisor configuration 20 | 21 | echo "generating supervisor config..." 22 | cd /opt/vespene 23 | sudo $PYTHON manage.py generate_supervisor --path /etc/vespene/supervisord.conf --workers "$WORKER_CONFIG" --executable=$PYTHON --source /opt/vespene --gunicorn "$GUNICORN_OPTS" 24 | echo "creating init script..." 25 | sudo chown -R $APP_USER /etc/vespene/ 26 | 27 | # -- 28 | 29 | # bail if on a system not supporting the systemd setup below 30 | if [ "$DISTRO" == "MacOS" ]; then 31 | echo "launch supervisor with: supervisord -n c /etc/vespene/supervisord.conf" 32 | exit 0 33 | elif [ "$DISTRO" == "alpine" ]; then 34 | echo "launch supervisor with: supervisord -n -c /etc/vespene/supervisord.conf" 35 | exit 0 36 | elif [ "$DISTRO" == "amazon" ]; then 37 | echo "launch supervisor with: sudo -u $APP_USER PATH=/usr/local/bin:\$PATH /usr/local/bin/supervisord -n -c /etc/vespene/supervisord.conf" 38 | exit 0 39 | fi 40 | 41 | 42 | # --- 43 | 44 | # generate systemd init script 45 | 46 | sudo tee /etc/systemd/system/vespene.service >/dev/null <{{ path }} 5 | 6 | {% for file in files %} 7 | 8 | 15 | 16 | {% endfor %} 17 |
9 | {% if file['directory'] %} 10 | {{ file['basename'] }} 11 | {% else %} 12 | {{ file['basename'] }} 13 | {% endif %} 14 |
18 | 19 | {% endblock %} 20 | 21 | 22 | -------------------------------------------------------------------------------- /vespene/jinja2/generic_edit.j2: -------------------------------------------------------------------------------- 1 | {% extends "theme.j2" %} 2 | 3 | {% block content %} 4 |
5 | 6 | {% set can_edit = False %} 7 | {% if flavor == 'Detail' %} 8 | {% if permissions.check_can_edit(obj, request) %} 9 | {% set can_edit = True %} 10 | {% endif %} 11 |

{{ object_label }}: {{ obj }}

12 | {% elif flavor == 'New' %} 13 |

Create {{ object_label }}

14 | {% elif flavor == 'Edit' %} 15 | {% if permissions.check_can_edit(obj, request) %} 16 | {% set can_edit = True %} 17 | {% endif %} 18 |

Edit {{ object_label }}: {{ obj }}

19 | {% endif %} 20 |
21 | 22 | 23 | {% if flavor == 'Detail' %} 24 | {{ crispy(form.make_read_only()) }} 25 | {% else %} 26 | {{ crispy(form) }} 27 | {% endif %} 28 |
29 | {% if flavor == 'New' or (flavor == 'Edit' and can_edit) %} 30 | 31 | {% elif flavor == 'Detail' and can_edit %} 32 | Edit 33 | {% endif %} 34 |
35 | {% endblock %} 36 | 37 | {% block extra_javascript %} 38 | 39 | $(document).ready(function() { 40 | no_validate_form_hack() 41 | }); 42 | 43 | {% endblock %} 44 | 45 | 46 | 47 | 48 | -------------------------------------------------------------------------------- /vespene/jinja2/generic_list.j2: -------------------------------------------------------------------------------- 1 | {% extends "theme.j2" %} 2 | 3 | {% block content %} 4 | 5 | 6 | 7 | 8 | {% for col in extra_columns %} 9 | 10 | {% endfor %} 11 | {% if supports_delete %} 12 | 13 | {% endif %} 14 | 15 | 16 | 17 | {% for obj in objects %} 18 | 19 | 28 | 29 | {% for col in extra_columns %} 30 | 31 | {% endfor %} 32 | 33 | {% if supports_delete %} 34 | 39 | {% endif %} 40 | 41 | 42 | {% endfor %} 43 |
{{ object_label }}{{ col[0] }}Delete
20 | {% if supports_edit %} 21 | 22 | {% else %} 23 | 24 | {% endif %} 25 | {{ name_cell(obj) | safe }} 26 | 27 | {{ col[1](obj) | safe }} 35 | 36 | 37 | 38 |
44 | 45 | 60 | 61 |
62 | {% if supports_new %} 63 | New {{ object_label }} 65 | 66 | {% endif %} 67 |
68 | 69 | {% endblock %} 70 | 71 | 72 | -------------------------------------------------------------------------------- /vespene/jinja2/generic_new.j2: -------------------------------------------------------------------------------- 1 | {% extends "generic_edit.j2" %} -------------------------------------------------------------------------------- /vespene/jinja2/index.j2: -------------------------------------------------------------------------------- 1 | {% extends "theme.j2" %} 2 | 3 | {% block content %} 4 |

5 | Welcome, {{ user }}. 6 |

7 | {% endblock %} -------------------------------------------------------------------------------- /vespene/jinja2/pipeline_map.j2: -------------------------------------------------------------------------------- 1 | {% extends "theme.j2" %} 2 | 3 | {% block content %} 4 | 5 |

Pipeline: {{ pipeline.name}}

6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | {% for stage in stages %} 15 | 16 | 19 | {% for project in projects_by_stage[stage.name] %} 20 | 23 | {% endfor %} 24 | 25 | {% endfor %} 26 |
StageProjects
17 | {{ stage.name }} 18 | 21 | {{ describe_project(project) | safe }} 22 |
27 | 28 | 29 | 30 | {% endblock %} 31 | 32 | {% block extra_javascript %} 33 | {% endblock %} -------------------------------------------------------------------------------- /vespene/jinja2/project_start_prompt.j2: -------------------------------------------------------------------------------- 1 | {% extends "theme.j2" %} 2 | 3 | {% block content %} 4 |
5 | 6 | 7 |

Start {{ object_label }}: {{ obj }}

8 | 9 | Before you can start this project you must answer the following configured questions. 10 | 11 |
12 | 13 | 14 | 15 | 16 | {% for question in questions %} 17 | 18 | {% set question_prompt = question.get('prompt', 'invalid') %} 19 | {% set question_type = question.get('type', 'text') %} 20 | {% set question_variable = question.get('variable', 'invalid') %} 21 | {% set question_choices = question.get('choices', []) %} 22 | 23 | 24 | {% if question_prompt != 'invalid' %} 25 | 26 | {% else %} 27 | 28 | {% endif %} 29 | 61 | 62 |
{{ question_prompt }}{{ question_variable }} 30 | 31 | {% if question_variable == 'invalid' %} 32 | This question is malformed. Please specify 'variable' in the configuration. 33 | {% else %} 34 | {% if question_type == 'text' %} 35 | 36 | {% elif question_type == 'radio' %} 37 | {% if (question_choices | length) < 1 %} 38 | This question is malformed. Please specify 'choices' in the configuration. 39 | {% else %} 40 | {% for choice in question_choices %} 41 | {{ choice }}
42 | {% endfor %} 43 | {% endif %} 44 | {% elif question_type == 'multi' %} 45 | {% if (question_choices | length) < 1 %} 46 | This question is malformed. Please specify 'choices' in the configuration. 47 | {% else %} 48 |
49 | {% for choice in question_choices %} 50 | {{ choice }}
51 | {% endfor %} 52 |
53 | {% endif %} 54 | {% else %} 55 | This question is malformed. The question type '{{ question_type }}' is not supported. 56 | {% endif %} 57 | {% endif %} 58 | {% endfor %} 59 | 60 |
63 | 64 |
65 | 66 | {% endblock %} -------------------------------------------------------------------------------- /vespene/jinja2/registration/login.html: -------------------------------------------------------------------------------- 1 | {% extends "./theme.j2" %} 2 | 3 | {% block title %}Login{% endblock %} 4 | 5 | {% block content %} 6 |

Login

7 |
8 | 9 | {{ form.as_p() | safe }} 10 | 11 |
12 | {% endblock %} -------------------------------------------------------------------------------- /vespene/jinja2/theme.j2: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 26 | 27 | {% block title %}Vespene{% endblock %} 28 | 29 | 30 | 31 | 32 |
33 | 34 | 64 | 65 | 66 | 67 |
68 | 69 | {% if messages %} 70 |
    71 | {% for msg in messages %} 72 | 75 | {% endfor %} 76 |
77 | {% endif %} 78 | 79 | {% block content %} 80 | {% endblock %} 81 |
82 | 83 |
84 | 85 |
86 |
87 | {% if version is defined %} 88 |
Vespene {{ version }}. (C) 2018 Michael DeHaan LLC + Contributors.
89 | {% endif %} 90 | 91 | 92 | 93 | 94 | -------------------------------------------------------------------------------- /vespene/management/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018, Michael DeHaan LLC 2 | # License: Apache License Version 2.0 -------------------------------------------------------------------------------- /vespene/management/commands/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018, Michael DeHaan LLC 2 | # License: Apache License Version 2.0 -------------------------------------------------------------------------------- /vespene/management/commands/autoscaler.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018, Michael DeHaan LLC 2 | # License: Apache License Version 2.0 3 | # ----------------------------------------------------- 4 | # runs autoscaling plugins to dynamically size worker pools 5 | 6 | import time 7 | import traceback 8 | import subprocess 9 | import sys 10 | from datetime import datetime 11 | 12 | from django.core.management.base import BaseCommand, CommandError 13 | from django.conf import settings 14 | from django.utils import timezone 15 | 16 | from vespene.models.worker_pool import WorkerPool 17 | from vespene.models.build import Build 18 | from vespene.common.logger import Logger 19 | from vespene.common.plugin_loader import PluginLoader 20 | 21 | LOG = Logger() 22 | 23 | class Command(BaseCommand): 24 | help = 'Runs autoscaling logic for one or more configured worker pools' 25 | 26 | def add_arguments(self, parser): 27 | parser.add_argument('--queue', action='append', type=str, help='name of the queue, use \'general\' for the unassigned queue') 28 | parser.add_argument('--sleep', type=int, help='how long to sleep between checks (in seconds)', default=20) 29 | parser.add_argument('--force', action='store_true', help='ignore timers and run the detector, then exit') 30 | 31 | def get_worker_pool(self, pool_name): 32 | 33 | worker_pools = WorkerPool.objects.filter(name=pool_name) 34 | if not worker_pools.exists(): 35 | LOG.info("worker pool does not exist: %s" % pool_name) 36 | return None 37 | return worker_pools.first() 38 | 39 | def handle_pool(self, worker_pool, planner, executor, force): 40 | 41 | if worker_pool is None: 42 | LOG.warning("there is no worker pool named %s yet" % worker_pool) 43 | # probably a provisioning order issue, this will degrade performance but should not be fatal 44 | # just avoid hammering the system until it exists 45 | time.sleep(60) 46 | return 47 | 48 | if not worker_pool.autoscaling_enabled: 49 | return 50 | 51 | now = datetime.now(tz=timezone.utc) 52 | autoscale_status = 0 53 | last_autoscaled = worker_pool.last_autoscaled 54 | 55 | try: 56 | if not (force or planner.is_time_to_adjust(worker_pool)): 57 | return 58 | 59 | parameters = planner.get_parameters(worker_pool) 60 | LOG.debug("autoscaling parameters: %s for %s" % (parameters, worker_pool.name)) 61 | 62 | result = executor.scale_worker_pool(worker_pool, parameters) 63 | 64 | LOG.info("autoscaling success for %s" % worker_pool.name) 65 | last_autoscaled = datetime.now(tz=timezone.utc) 66 | 67 | except subprocess.CalledProcessError as cpe: 68 | 69 | LOG.error("autoscaling failed, return code: %s" % cpe.returncode) 70 | autoscale_status = cpe.returncode 71 | 72 | except: 73 | 74 | traceback.print_exc() 75 | LOG.error("autoscaling failed for %s" % worker_pool.name) 76 | autoscale_status = 1 77 | 78 | finally: 79 | WorkerPool.objects.filter( 80 | pk=worker_pool.pk 81 | ).update(last_autoscaled=last_autoscaled, autoscaling_status=autoscale_status) 82 | 83 | def handle(self, *args, **options): 84 | 85 | worker_pools = options.get('queue') 86 | sleep_time = options.get('sleep') 87 | force = options.get('force') 88 | 89 | LOG.info("started...") 90 | 91 | self.plugin_loader = PluginLoader() 92 | self.planner_plugins = self.plugin_loader.get_autoscaling_planner_plugins() 93 | self.executor_plugins = self.plugin_loader.get_autoscaling_executor_plugins() 94 | 95 | while True: 96 | 97 | for pool in worker_pools: 98 | worker_pool = self.get_worker_pool(pool) 99 | if worker_pool is None: 100 | continue 101 | planner = self.planner_plugins[worker_pool.planner] 102 | executor = self.executor_plugins[worker_pool.executor] 103 | self.handle_pool(worker_pool, planner, executor, force) 104 | time.sleep(1) 105 | 106 | if force: 107 | break 108 | else: 109 | time.sleep(sleep) 110 | 111 | LOG.info("exited...") 112 | 113 | -------------------------------------------------------------------------------- /vespene/management/commands/cleanup.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018, Michael DeHaan LLC 2 | # License: Apache License Version 2.0 3 | 4 | import glob 5 | import os.path 6 | import shutil 7 | from datetime import datetime, timedelta 8 | 9 | from django.utils import timezone 10 | from django.core.management.base import BaseCommand, CommandError 11 | from django.conf import settings 12 | 13 | # from vespene.models.project import Project 14 | from vespene.models.build import Build 15 | from vespene.common.logger import Logger 16 | 17 | LOG = Logger() 18 | 19 | class Command(BaseCommand): 20 | help = 'Removes old builds and buildroots' 21 | 22 | def add_arguments(self, parser): 23 | 24 | parser.add_argument( 25 | '--remove-builds', 26 | action='store_true', 27 | dest='remove_builds', 28 | help='If set, clean up old builds', 29 | ) 30 | parser.add_argument( 31 | '--remove-build-roots', 32 | action='store_true', 33 | dest='remove_build_roots', 34 | help='If set, clean up old build roots', 35 | ) 36 | parser.add_argument( 37 | '--days', 38 | dest='days', 39 | help='Cleanup items older than this many days', 40 | default=-1, 41 | ) 42 | 43 | def handle(self, *args, **options): 44 | 45 | 46 | remove_builds = options['remove_builds'] 47 | remove_build_roots = options['remove_build_roots'] 48 | days = int(options['days']) 49 | 50 | if not remove_builds and not remove_build_roots: 51 | raise CommandError("expecting at least one of: --remove-builds or --remove-build-roots") 52 | if days < 0: 53 | raise CommandError("--days is required") 54 | 55 | threshold = datetime.now(tz=timezone.utc) - timedelta(days=days) 56 | timestamp = threshold.timestamp() 57 | 58 | if remove_build_roots: 59 | build_root = settings.BUILD_ROOT 60 | if not os.path.exists(build_root): 61 | raise CommandError("BUILD_ROOT as configured in settings (%s) does not exist on this node" % build_root) 62 | contents = glob.glob(os.path.join(build_root, "*")) 63 | count = 0 64 | for dirname in contents: 65 | if os.path.isdir(dirname): 66 | mtime = os.path.getmtime(dirname) 67 | if mtime < timestamp: 68 | count = count + 1 69 | shutil.rmtree(dirname) 70 | print("Deleted %d build roots" % count) 71 | 72 | if remove_builds: 73 | builds = Build.objects.filter( 74 | queued_time__gt = threshold 75 | ) 76 | count = builds.count() 77 | builds.delete() 78 | print("Deleted %d builds" % count) 79 | 80 | 81 | -------------------------------------------------------------------------------- /vespene/management/commands/generate_secret.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018, Michael DeHaan LLC 2 | # License: Apache License Version 2.0 3 | 4 | import sys 5 | import os 6 | import random 7 | from cryptography import fernet 8 | from django.core.management.base import BaseCommand 9 | from vespene.common.logger import Logger 10 | 11 | CONTENTS = """ 12 | DJANGO_SECRET_KEY = "%s" 13 | 14 | SYMETRIC_SECRET_KEY = %s 15 | """ 16 | 17 | # /usr/bin/python36 manage.py supervisor_generate --path /etc/vespene/supervisord.conf --controller true --workers "name1=2 name2=5" --python /usr/bin/python36 18 | 19 | LOG = Logger() 20 | 21 | SECRETS = "/etc/vespene/settings.d/secrets.py" 22 | 23 | def django_secret_key(): 24 | return ''.join(random.SystemRandom().choice('abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*(-_=+)') for i in range(50)) 25 | 26 | def symetric_secret_key(): 27 | return fernet.Fernet.generate_key() 28 | 29 | class Command(BaseCommand): 30 | help = 'Configures supervisor for production deploys. See the setup/ directory for example usage' 31 | 32 | def add_arguments(self, parser): 33 | pass 34 | 35 | def handle(self, *args, **options): 36 | 37 | if os.path.exists(SECRETS): 38 | print("WARNING: running this command again would render some secrets in the database unreadable") 39 | print("if you wish to proceed, delete %s manually first" % SECRETS) 40 | sys.exit(1) 41 | 42 | fd = open(SECRETS, "w+") 43 | s1 = django_secret_key() 44 | s2 = symetric_secret_key() 45 | fd.write(CONTENTS % (s1, s2)) 46 | fd.close() 47 | 48 | 49 | 50 | -------------------------------------------------------------------------------- /vespene/management/commands/generate_supervisor.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018, Michael DeHaan LLC 2 | # License: Apache License Version 2.0 3 | #--------------------------------------------------------------------------- 4 | # generate_supervisor.py - CLI command that is used by the setup shell 5 | # scripts as mentioned in the web documentation to more easily generate 6 | # a supervisord configuration that runs both the Vespene web server and 7 | # any number of worker processes. For usage see setup/centos7/6_services.py. 8 | #--------------------------------------------------------------------------- 9 | 10 | from django.core.management.base import BaseCommand, CommandError 11 | from vespene.common.logger import Logger 12 | 13 | import os 14 | 15 | PREAMBLE = """ 16 | [supervisord] 17 | childlogdir=/var/log/vespene/ 18 | logfile=/var/log/vespene/supervisord.log 19 | logfile_maxbytes=50MB 20 | logfile_backups=10 21 | loglevel=info 22 | pidfile=/var/run/supervisord.pid 23 | nodaemon=false 24 | minfds=1024 25 | minprocs=200 26 | """ 27 | 28 | WEB = """ 29 | [program:server] 30 | command=gunicorn %s vespene.wsgi 31 | process_name=%s 32 | directory=%s 33 | autostart=true 34 | autorestart=false 35 | redirect_stderr=true 36 | stdout_logfile = /var/log/vespene/web.log 37 | stdout_logfile_maxbytes=50MB 38 | """ 39 | 40 | WORKER = """ 41 | [program:worker_%s] 42 | command=/usr/bin/ssh-agent %s manage.py worker %s 43 | numprocs=%s 44 | process_name=%s 45 | directory=%s 46 | autostart=true 47 | autorestart=false 48 | redirect_stderr=true 49 | stdout_logfile = /var/log/vespene/worker_%s.log 50 | stdout_logfile_maxbytes=50MB 51 | """ 52 | 53 | # USAGE: manage.py generate_supervisor --path /etc/vespene/supervisord.conf --workers "name1=2 name2=5" \ 54 | # --executable `which python` --source=/opt/vespene --gunicorn "--bind 127.0.0.1:8000" 55 | LOG = Logger() 56 | 57 | class Command(BaseCommand): 58 | help = 'Configures supervisor for production deploys. See the setup/ directory for example usage' 59 | 60 | def add_arguments(self, parser): 61 | parser.add_argument('--path', type=str, help='filename to write') 62 | parser.add_argument('--workers', type=str, help="what workers to run?") 63 | parser.add_argument('--executable', type=str, help="python executable") 64 | parser.add_argument('--source', type=str, help='source') 65 | parser.add_argument('--gunicorn', type=str, help='gnuicorn options string', default='--bind 127.0.0.1:8000') 66 | 67 | def handle(self, *args, **options): 68 | 69 | path = options.get('path', None) 70 | workers = options.get('workers', None) 71 | python = options.get('executable', None) 72 | source = options.get('source', None) 73 | gunicorn = options.get('gunicorn', None) 74 | 75 | if not workers and not "=" in workers: 76 | raise CommandError("worker configuration does not look correct") 77 | if not python or not os.path.exists(python): 78 | raise CommandError("--executable must point to an interpreter") 79 | 80 | workers = workers.split(" ") 81 | 82 | fd = open(path, "w+") 83 | fd.write(PREAMBLE) 84 | fd.write("\n") 85 | fd.write(WEB % (gunicorn, "%(program_name)s", source)) 86 | fd.write("\n") 87 | 88 | for worker in workers: 89 | tokens = worker.split("=", 1) 90 | key = tokens[0] 91 | value = tokens[1] 92 | stanza = WORKER % (key, python, key, value, "%(program_name)s%(process_num)s", source, key) 93 | fd.write(stanza) 94 | fd.write("\n") 95 | 96 | fd.close() 97 | 98 | -------------------------------------------------------------------------------- /vespene/management/commands/project_control.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018, Michael DeHaan LLC 2 | # License: Apache License Version 2.0 3 | #--------------------------------------------------------------------------- 4 | # project_control.py - CLI command that can start or stop projects. This is 5 | # mostly used for development testing and can be upgraded. 6 | #--------------------------------------------------------------------------- 7 | 8 | from django.core.management.base import BaseCommand, CommandError 9 | from vespene.models.project import Project 10 | from vespene import jobkick 11 | from vespene.common.logger import Logger 12 | 13 | LOG = Logger() 14 | 15 | class Command(BaseCommand): 16 | help = 'Starts and stops builds for a given project' 17 | 18 | def add_arguments(self, parser): 19 | parser.add_argument('project', type=int, help='project id') 20 | 21 | parser.add_argument( 22 | '--start', 23 | action='store_true', 24 | dest='start', 25 | help='Start a build for this project', 26 | ) 27 | parser.add_argument( 28 | '--stop', 29 | action='store_true', 30 | dest='stop', 31 | help='Stop a build for this project', 32 | ) 33 | 34 | def handle(self, *args, **options): 35 | 36 | 37 | project_id = options['project'] 38 | start = options['start'] 39 | stop = options['stop'] 40 | 41 | project = Project.objects.get(id=project_id) 42 | 43 | if not start and not stop: 44 | raise CommandError("either --start or --stop are required") 45 | if stop and start: 46 | raise CommandError("--start and --stop are mutually exclusive") 47 | elif start: 48 | LOG.info("Starting project") 49 | jobkick.start_project(project) 50 | else: 51 | LOG.info("Stopping project") 52 | jobkick.stop_project(project) 53 | -------------------------------------------------------------------------------- /vespene/management/commands/worker.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018, Michael DeHaan LLC 2 | # License: Apache License Version 2.0 3 | #--------------------------------------------------------------------------- 4 | # worker.py - this is the management command that starts worker processes 5 | # that work a given queue, and is typically involved by supervisor, configured 6 | # in /etc/vespene/supervisord.conf 7 | #--------------------------------------------------------------------------- 8 | 9 | from django.core.management.base import BaseCommand 10 | from vespene.workers.daemon import Daemon 11 | 12 | 13 | 14 | class Command(BaseCommand): 15 | help = 'Starts a daemon process for background jobs' 16 | 17 | def add_arguments(self, parser): 18 | parser.add_argument('--max-builds', dest="max_builds", type=int, help='if set, terminate after this many builds', default=-1) 19 | parser.add_argument('--max-wait-minutes', dest="max_wait_minutes", type=int, help="if set, terminate after this many minutes of no queued builds", default=-1) 20 | parser.add_argument( '--build-id', dest="build_id", type=int, help='run only this one build ID then exit', default=-1) 21 | 22 | parser.add_argument('queue', type=str, help='name of the queue, use \'general\' for the unassigned queue') 23 | 24 | def handle(self, *args, **options): 25 | queue = options['queue'] 26 | max_wait_minutes = options['max_wait_minutes'] 27 | max_builds = options['max_builds'] 28 | build_id = options['build_id'] 29 | worker = Daemon(queue, 30 | max_builds=max_builds, 31 | max_wait_minutes=max_wait_minutes, 32 | build_id=build_id) 33 | worker.run() 34 | 35 | 36 | -------------------------------------------------------------------------------- /vespene/manager/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018, Michael DeHaan LLC 2 | # License: Apache License Version 2.0 3 | 4 | 5 | class Shared(object): 6 | __shared_state = {} 7 | def __init__(self): 8 | self.__dict__ = self.__shared_state 9 | 10 | -------------------------------------------------------------------------------- /vespene/manager/output.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018, Michael DeHaan LLC 2 | # License: Apache License Version 2.0 3 | # ------------------------------------------------------------------------- 4 | # output.py - applies all output plugins to a message. Output plugins 5 | # can be used to format build output or also redirect it to additional 6 | # sources. 7 | # -------------------------------------------------------------------------- 8 | 9 | from vespene.common.logger import Logger 10 | from vespene.common.plugin_loader import PluginLoader 11 | 12 | LOG = Logger() 13 | 14 | class OutputManager(object): 15 | 16 | def __init__(self): 17 | self.plugin_loader = PluginLoader() 18 | self.plugins = self.plugin_loader.get_output_plugins() 19 | 20 | def get_msg(self, build, msg): 21 | for p in self.plugins: 22 | msg = p.filter(build, msg) 23 | return msg 24 | -------------------------------------------------------------------------------- /vespene/manager/permissions.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018, Michael DeHaan LLC 2 | # License: Apache License Version 2.0 3 | # ------------------------------------------------------------------------- 4 | # permissions.py - central point for all code to use to ask questions about 5 | # what things are allowed. Just returns yes/no, does not raise exceptions. 6 | # Implemented by deferring to plugins in 'plugins/authorization' as configured 7 | # in settings. 8 | # -------------------------------------------------------------------------- 9 | 10 | from vespene.common.logger import Logger 11 | from vespene.common.plugin_loader import PluginLoader 12 | 13 | LOG = Logger() 14 | 15 | class PermissionsManager(object): 16 | 17 | def __init__(self): 18 | self.plugin_loader = PluginLoader() 19 | self.plugins = self.plugin_loader.get_authorization_plugins() 20 | 21 | def _all_true(self, method, subject, request, *args, **kwargs): 22 | for plugin in self.plugins: 23 | fn = getattr(plugin, method) 24 | if not fn(subject, request, *args, **kwargs): 25 | return False 26 | return True 27 | 28 | def filter_queryset_for_list(self, qs, request, *args, **kwargs): 29 | result_qs = qs 30 | for plugin in self.plugins: 31 | result_qs = plugin.filter_queryset_for_list(result_qs, request, *args, **kwargs) 32 | return result_qs 33 | 34 | def filter_queryset_for_view(self, qs, request, *args, **kwargs): 35 | result_qs = qs 36 | for plugin in self.plugins: 37 | result_qs = plugin.filter_queryset_for_view(result_qs, request, *args, **kwargs) 38 | return result_qs 39 | 40 | def filter_queryset_for_delete(self, qs, request, *args, **kwargs): 41 | result_qs = qs 42 | for plugin in self.plugins: 43 | result_qs = plugin.filter_queryset_for_delete(result_qs, request, *args, **kwargs) 44 | return result_qs 45 | 46 | def filter_queryset_for_edit(self, qs, request, *args, **kwargs): 47 | result_qs = qs 48 | for plugin in self.plugins: 49 | result_qs = plugin.filter_queryset_for_edit(result_qs, request, *args, **kwargs) 50 | return result_qs 51 | 52 | def check_can_view(self, obj, request, *args, **kwargs): 53 | return self._all_true('check_can_view', obj, request, *args, **kwargs) 54 | 55 | def check_can_edit(self, obj, request, *args, **kwargs): 56 | return self._all_true('check_can_edit', obj, request, *args, **kwargs) 57 | 58 | def check_can_delete(self, obj, request, *args, **kwargs): 59 | return self._all_true('check_can_delete', obj, request, *args, **kwargs) 60 | 61 | def check_can_create(self, cls, request, *args, **kwargs): 62 | return self._all_true('check_can_create', cls, request, *args, **kwargs) 63 | 64 | def check_can_start(self, obj, request, *args, **kwargs): 65 | return self._all_true('check_can_start', obj, request, *args, **kwargs) 66 | 67 | def check_can_stop(self, obj, request, *args, **kwargs): 68 | return self._all_true('check_can_view', obj, request, *args, **kwargs) 69 | 70 | -------------------------------------------------------------------------------- /vespene/manager/secrets.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018, Michael DeHaan LLC 2 | # License: Apache License Version 2.0 3 | # ------------------------------------------------------------------------- 4 | # secrets.py - uses plugins to symetrically encrypt/decrypt content 5 | # in the database. All plugins in the configuration can be used to 6 | # decode data (if they match), but only the first plugin the settings 7 | # configuration will be used to encode. 8 | # -------------------------------------------------------------------------- 9 | 10 | from vespene.common.logger import Logger 11 | from vespene.common.plugin_loader import PluginLoader 12 | 13 | LOG = Logger() 14 | 15 | HEADER = "[VESPENE-CLOAKED]" 16 | 17 | class SecretsManager(object): 18 | 19 | def __init__(self): 20 | self.plugin_loader = PluginLoader() 21 | self.plugins = self.plugin_loader.get_secrets_plugins() 22 | 23 | def cloak(self, msg): 24 | if len(self.plugins) == 0: 25 | return msg 26 | if not self.is_cloaked(msg): 27 | return self.plugins[0].cloak(msg) 28 | else: 29 | # already cloaked, will not re-cloak 30 | return msg 31 | 32 | def is_cloaked(self, msg): 33 | return msg.startswith(HEADER) 34 | 35 | def decloak(self, msg): 36 | remainder = msg.replace(HEADER, "", 1) 37 | for plugin in self.plugins: 38 | if plugin.recognizes(remainder): 39 | return plugin.decloak(remainder) 40 | return remainder 41 | -------------------------------------------------------------------------------- /vespene/manager/webhooks.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018, Michael DeHaan LLC 2 | # License: Apache License Version 2.0 3 | # ------------------------------------------------------------------------- 4 | # webhooks.py - implementation of webhook support. Right now this isn't 5 | # pluggable and contains an implementation tested for GitHub and some 6 | # code that *probably* works for GitLab. Upgrades welcome. 7 | # -------------------------------------------------------------------------- 8 | 9 | import json 10 | from vespene.common.logger import Logger 11 | from vespene.manager.jobkick import start_project 12 | from vespene.models.project import Project 13 | 14 | LOG = Logger() 15 | 16 | # =============================================================================================== 17 | 18 | class Webhooks(object): 19 | 20 | def __init__(self, request, token): 21 | 22 | body = request.body.decode('utf-8') 23 | self.token = token 24 | self.content = json.loads(body) 25 | 26 | def handle(self): 27 | """ 28 | Invoked by views.py, recieves all webhooks and attempts to find out what 29 | projects correspond with them by looking at the repo. If the project 30 | is webhook enabled, it will create a QUEUED build for that project. 31 | """ 32 | 33 | # FIXME: at some point folks will want to support testing commits on branches 34 | # not set on the project. This is a good feature idea, and to do this we 35 | # should add a webhooks_trigger_any_branch type option, that creates a build 36 | # with any incoming branch specified. 37 | 38 | possibles = [] 39 | 40 | # this fuzzy code looks for what may come in for webhook JSON for GitHub and GitLab 41 | # extension to support other SCM webhooks is welcome. 42 | for section in [ 'project', 'repository' ]: 43 | if section in self.content: 44 | for key in [ 'git_url', 'ssh_url', 'clone_url', 'git_ssh_url', 'git_http_url' ]: 45 | repo = self.content[section].get(key, None) 46 | if repo is not None: 47 | possibles.append(repo) 48 | 49 | # find projects that match repos we have detected 50 | qs = Project.objects.filter(webhook_enabled=True, repo_url__in=possibles) 51 | for project in qs: 52 | if project.webhook_token is None or project.webhook_token == self.token: 53 | LOG.info("webhook starting project: %s" % project.id) 54 | if project.repo_branch is not None: 55 | ref = self.content.get('ref') 56 | if ref: 57 | branch = ref.split("/")[-1] 58 | # if the project selects a particular branch and this repo is for 59 | # a different branch, we'll ignore the webhook. Otherwise we'll 60 | # assume there is only one branch. This could result in too many 61 | # builds - if we add special handling, we should consider SVN 62 | # doesn't really have branches 63 | if project.repo_branch and branch and branch != project.repo_branch: 64 | LOG.info("skipping, references another branch") 65 | continue 66 | start_project(project) 67 | -------------------------------------------------------------------------------- /vespene/migrations/0002_auto_20181027_1530.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.1.2 on 2018-10-27 15:30 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ('vespene', '0001_initial'), 10 | ] 11 | 12 | operations = [ 13 | migrations.AlterField( 14 | model_name='build', 15 | name='is_internal', 16 | field=models.BooleanField(blank=True, default=False), 17 | ), 18 | migrations.AlterField( 19 | model_name='organization', 20 | name='description', 21 | field=models.TextField(blank=True), 22 | ), 23 | ] 24 | -------------------------------------------------------------------------------- /vespene/migrations/0003_auto_20181101_2224.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.1.2 on 2018-11-01 22:24 2 | 3 | from django.db import migrations, models 4 | import django.db.models.deletion 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ('vespene', '0002_auto_20181027_1530'), 11 | ] 12 | 13 | operations = [ 14 | migrations.AlterField( 15 | model_name='project', 16 | name='scm_login', 17 | field=models.ForeignKey(blank=True, help_text='... or leave this blank and add an SSH key', null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='projects', to='vespene.ServiceLogin'), 18 | ), 19 | migrations.AlterField( 20 | model_name='workerpool', 21 | name='sudo_password', 22 | field=models.CharField(blank=True, max_length=1024, null=True), 23 | ), 24 | ] 25 | -------------------------------------------------------------------------------- /vespene/migrations/0004_auto_20181105_2006.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.1.2 on 2018-11-05 20:06 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ('vespene', '0003_auto_20181101_2224'), 10 | ] 11 | 12 | operations = [ 13 | migrations.AddField( 14 | model_name='workerpool', 15 | name='autoscaling_enabled', 16 | field=models.BooleanField(blank=True, default=False), 17 | ), 18 | migrations.AddField( 19 | model_name='workerpool', 20 | name='autoscaling_status', 21 | field=models.IntegerField(blank=True, null=True), 22 | ), 23 | migrations.AddField( 24 | model_name='workerpool', 25 | name='excess', 26 | field=models.IntegerField(blank=True, default=0), 27 | ), 28 | migrations.AddField( 29 | model_name='workerpool', 30 | name='executor', 31 | field=models.CharField(blank=True, max_length=512, null=True), 32 | ), 33 | migrations.AddField( 34 | model_name='workerpool', 35 | name='executor_command', 36 | field=models.CharField(blank=True, default='', max_length=1024), 37 | ), 38 | migrations.AddField( 39 | model_name='workerpool', 40 | name='last_autoscaled', 41 | field=models.DateTimeField(blank=True, null=True), 42 | ), 43 | migrations.AddField( 44 | model_name='workerpool', 45 | name='maximum', 46 | field=models.IntegerField(blank=True, default=10), 47 | ), 48 | migrations.AddField( 49 | model_name='workerpool', 50 | name='minimum', 51 | field=models.IntegerField(blank=True, default=0), 52 | ), 53 | migrations.AddField( 54 | model_name='workerpool', 55 | name='multiplier', 56 | field=models.IntegerField(blank=True, default=1), 57 | ), 58 | migrations.AddField( 59 | model_name='workerpool', 60 | name='planner', 61 | field=models.CharField(blank=True, max_length=512, null=True), 62 | ), 63 | migrations.AddField( 64 | model_name='workerpool', 65 | name='queued_weight', 66 | field=models.IntegerField(blank=True, default=1), 67 | ), 68 | migrations.AddField( 69 | model_name='workerpool', 70 | name='reevaluate_minutes', 71 | field=models.IntegerField(blank=True, default=10), 72 | ), 73 | migrations.AddField( 74 | model_name='workerpool', 75 | name='running_weight', 76 | field=models.IntegerField(blank=True, default=1), 77 | ), 78 | ] 79 | -------------------------------------------------------------------------------- /vespene/migrations/0005_auto_20181105_2042.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.1.2 on 2018-11-05 20:42 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ('vespene', '0004_auto_20181105_2006'), 10 | ] 11 | 12 | operations = [ 13 | migrations.AlterField( 14 | model_name='workerpool', 15 | name='queued_weight', 16 | field=models.FloatField(blank=True, default=1.0), 17 | ), 18 | migrations.AlterField( 19 | model_name='workerpool', 20 | name='running_weight', 21 | field=models.FloatField(blank=True, default=1.0), 22 | ), 23 | ] 24 | -------------------------------------------------------------------------------- /vespene/migrations/0006_auto_20181105_2044.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.1.2 on 2018-11-05 20:44 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ('vespene', '0005_auto_20181105_2042'), 10 | ] 11 | 12 | operations = [ 13 | migrations.AlterField( 14 | model_name='workerpool', 15 | name='multiplier', 16 | field=models.FloatField(blank=True, default=1.0), 17 | ), 18 | ] 19 | -------------------------------------------------------------------------------- /vespene/migrations/0007_project_recursive.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.1.2 on 2018-11-06 21:37 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ('vespene', '0006_auto_20181105_2044'), 10 | ] 11 | 12 | operations = [ 13 | migrations.AddField( 14 | model_name='project', 15 | name='recursive', 16 | field=models.BooleanField(blank=True, default=True), 17 | ), 18 | ] 19 | -------------------------------------------------------------------------------- /vespene/migrations/0008_auto_20181106_2233.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.1.2 on 2018-11-06 22:33 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ('vespene', '0007_project_recursive'), 10 | ] 11 | 12 | operations = [ 13 | migrations.RemoveIndex( 14 | model_name='pipeline', 15 | name='pipeline_group_name_idx', 16 | ), 17 | migrations.RemoveField( 18 | model_name='pipeline', 19 | name='group_name', 20 | ), 21 | migrations.AddField( 22 | model_name='organization', 23 | name='allow_pipeline_definition', 24 | field=models.BooleanField(default=True), 25 | ), 26 | ] 27 | -------------------------------------------------------------------------------- /vespene/migrations/0009_remove_workerpool_sudo_password.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.1.2 on 2018-12-16 13:01 2 | 3 | from django.db import migrations 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ('vespene', '0008_auto_20181106_2233'), 10 | ] 11 | 12 | operations = [ 13 | migrations.RemoveField( 14 | model_name='workerpool', 15 | name='sudo_password', 16 | ), 17 | ] 18 | -------------------------------------------------------------------------------- /vespene/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vespene-io/_old_vespene/c0fbb217dfc5155bf974ab9e06fd43e539da63e0/vespene/migrations/__init__.py -------------------------------------------------------------------------------- /vespene/models/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018, Michael DeHaan LLC 2 | # License: Apache License Version 2.0 3 | # --------------------------------------------------------------------------- 4 | # models.py: base class of all DB models, some minor utility functions 5 | # models are smart objects that include behavior as well as representation 6 | # --------------------------------------------------------------------------- 7 | 8 | class BaseModel(object): 9 | 10 | def __str__(self): 11 | return self.name 12 | 13 | def as_dict(obj): 14 | if obj is None: 15 | return None 16 | else: 17 | return obj.as_dict() -------------------------------------------------------------------------------- /vespene/models/base.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018, Michael DeHaan LLC 2 | # License: Apache License Version 2.0 -------------------------------------------------------------------------------- /vespene/models/organization.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018, Michael DeHaan LLC 2 | # License: Apache License Version 2.0 3 | #--------------------------------------------------------------------------- 4 | # organization.py - a model of an organization like GitHub organizations 5 | # holding lots of repos for import 6 | #--------------------------------------------------------------------------- 7 | 8 | import json 9 | 10 | from django.contrib.auth.models import Group, User 11 | from django.db import models 12 | 13 | from vespene.manager import Shared 14 | from vespene.common.logger import Logger 15 | from vespene.models import BaseModel, as_dict 16 | from vespene.models.build import QUEUED, RUNNING, UNKNOWN 17 | from vespene.manager.permissions import PermissionsManager 18 | 19 | permissions = PermissionsManager() 20 | LOG = Logger() 21 | 22 | class Organization(models.Model, BaseModel): 23 | class Meta: 24 | db_table = 'organizations' 25 | indexes = [ 26 | models.Index(fields=['name'], name='organization_name_idx'), 27 | ] 28 | 29 | name = models.CharField(unique=True, max_length=512) 30 | description = models.TextField(blank=True) 31 | 32 | organization_type = models.CharField(max_length=100) 33 | organization_identifier = models.CharField(max_length=512, help_text="example: 'vespene-io' for github.com/vespene-io/") 34 | api_endpoint = models.CharField(max_length=512, blank=True, default="", help_text="blank, or https://{hostname}/api/v3 for GitHub Enterprise") 35 | 36 | import_enabled = models.BooleanField(default=True) 37 | 38 | import_without_dotfile = models.BooleanField(default=False) 39 | overwrite_project_name = models.BooleanField(default=True) 40 | overwrite_project_script = models.BooleanField(default=True) 41 | overwrite_configurations = models.BooleanField(default=True) 42 | allow_pipeline_definition = models.BooleanField(default=True) 43 | allow_worker_pool_assignment = models.BooleanField(default=True) 44 | auto_attach_ssh_keys = models.ManyToManyField('SshKey', related_name='+', blank=True, help_text="SSH keys to be assigned to imported projects") 45 | default_worker_pool = models.ForeignKey('WorkerPool', related_name='+', null=False, on_delete=models.PROTECT) 46 | 47 | force_rescan = models.BooleanField(default=False, help_text="rescan once at the next opportunity, ignoring refresh_minutes") 48 | refresh_minutes = models.IntegerField(default=120) 49 | scm_login = models.ForeignKey('ServiceLogin', related_name='organizations', on_delete=models.SET_NULL, null=True, help_text="... or add an SSH key in the next tab", blank=True) 50 | 51 | worker_pool = models.ForeignKey('WorkerPool', related_name='organizations', null=False, on_delete=models.PROTECT) 52 | 53 | created_by = models.ForeignKey(User, related_name='+', null=True, blank=True, on_delete=models.SET_NULL) 54 | 55 | last_build = models.ForeignKey('Build', null=True, blank=True, related_name='last_build_for_organization', on_delete=models.SET_NULL) 56 | active_build = models.ForeignKey('Build', null=True, blank=True, related_name='active_build_for_organization', on_delete=models.SET_NULL) 57 | last_successful_build = models.ForeignKey('Build', null=True, blank=True, related_name='last_successful_build_for_organization', on_delete=models.SET_NULL) 58 | 59 | def __str__(self): 60 | return self.name 61 | 62 | 63 | -------------------------------------------------------------------------------- /vespene/models/service_login.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018, Michael DeHaan LLC 2 | # License: Apache License Version 2.0 3 | #--------------------------------------------------------------------------- 4 | # service_login.py - a username/password combo, most likely for accessing 5 | # a SCM like git when SSH keys are not used. 6 | #--------------------------------------------------------------------------- 7 | 8 | from django.contrib.auth.models import Group, User 9 | from django.db import models 10 | from vespene.common.secrets import SecretsManager 11 | 12 | secrets = SecretsManager() 13 | 14 | from vespene.models import BaseModel 15 | 16 | class ServiceLogin(models.Model, BaseModel): 17 | 18 | class Meta: 19 | db_table = 'service_logins' 20 | indexes = [ 21 | models.Index(fields=['name'], name='service_login_name_idx'), 22 | ] 23 | 24 | name = models.CharField(unique=True, max_length=512) 25 | description = models.TextField(blank=True) 26 | username = models.CharField(max_length=512) 27 | password = models.CharField(max_length=512, blank=True) 28 | created_by = models.ForeignKey(User, related_name='+', null=True, blank=True, on_delete=models.SET_NULL) 29 | owner_groups = models.ManyToManyField(Group, related_name='service_logins', blank=True) 30 | 31 | def __str__(self): 32 | return self.name 33 | 34 | def save(self, *args, **kwargs): 35 | self.password = secrets.cloak(self.password) 36 | super().save(*args, **kwargs) 37 | 38 | def get_password(self): 39 | return secrets.decloak(self.password) 40 | 41 | -------------------------------------------------------------------------------- /vespene/models/snippet.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018, Michael DeHaan LLC 2 | # License: Apache License Version 2.0 3 | #--------------------------------------------------------------------------- 4 | # snippet.py - snippets are large chunks of text that are available as 5 | # jinja2 variables, but are unique in that they too are templated before 6 | # being made available as variables. A good way to reuse text between 7 | # multiple build scripts. 8 | #--------------------------------------------------------------------------- 9 | 10 | from django.contrib.auth.models import Group, User 11 | from django.db import models 12 | 13 | from vespene.models import BaseModel 14 | 15 | class Snippet(models.Model, BaseModel): 16 | 17 | class Meta: 18 | db_table = 'snippets' 19 | 20 | # TODO: assert that there are no spaces or characters not legal in python variables in the template name, using a custom validator / pre-save / etc. 21 | 22 | name = models.CharField(unique=True, max_length=512, help_text="Snippet names must be a valid python identifiers (ex: just_like_this1)") 23 | description = models.TextField(blank=True) 24 | text = models.TextField(help_text="this value will be used wherever the {{snippet_name}} appears in a build script") 25 | 26 | created_by = models.ForeignKey(User, related_name='+', null=True, blank=True, on_delete=models.SET_NULL) 27 | owner_groups = models.ManyToManyField(Group, related_name='snippets', blank=True) 28 | 29 | def __str__(self): 30 | return self.name 31 | -------------------------------------------------------------------------------- /vespene/models/ssh_key.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018, Michael DeHaan LLC 2 | # License: Apache License Version 2.0 3 | #--------------------------------------------------------------------------- 4 | # ssh_key.py - holds SSH private keys, with optional unlock passwords, 5 | # for use by SCM checkouts or SSH-based management of hosts when running 6 | # the build scripts 7 | #--------------------------------------------------------------------------- 8 | 9 | from django.contrib.auth.models import Group, User 10 | from django.db import models 11 | from vespene.common.secrets import SecretsManager 12 | from vespene.models import BaseModel 13 | from vespene.common.logger import Logger 14 | 15 | LOG = Logger() 16 | secrets = SecretsManager() 17 | 18 | class SshKey(models.Model, BaseModel): 19 | 20 | # NOTE: we don't support support SSH with password 21 | 22 | class Meta: 23 | db_table = 'ssh_keys' 24 | indexes = [ 25 | models.Index(fields=['name'], name='ssh_key_name_idx'), 26 | ] 27 | 28 | # TODO: FIXME: encrypt these in database using Django secrets 29 | name = models.CharField(unique=True, max_length=512) 30 | description = models.TextField(blank=True) 31 | private_key = models.TextField(blank=True) 32 | unlock_password = models.CharField(help_text="provide passphrase only for locked keys", max_length=512, blank=True) 33 | created_by = models.ForeignKey(User, related_name='+', null=True, blank=True, on_delete=models.SET_NULL) 34 | owner_groups = models.ManyToManyField(Group, related_name='ssh_keys',blank=True) 35 | 36 | def __str__(self): 37 | return self.name 38 | 39 | def save(self, *args, **kwargs): 40 | self.private_key = secrets.cloak(self.private_key) 41 | self.unlock_password = secrets.cloak(self.unlock_password) 42 | super().save(*args, **kwargs) 43 | 44 | def get_private_key(self): 45 | return secrets.decloak(self.private_key) 46 | 47 | def get_unlock_password(self): 48 | return secrets.decloak(self.unlock_password) -------------------------------------------------------------------------------- /vespene/models/stage.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018, Michael DeHaan LLC 2 | # License: Apache License Version 2.0 3 | #--------------------------------------------------------------------------- 4 | # stage.py - one of many steps of a CI/CD pipeline. See pipelines.py 5 | #--------------------------------------------------------------------------- 6 | 7 | from django.db import models 8 | 9 | from vespene.models import BaseModel 10 | 11 | class Stage(models.Model, BaseModel): 12 | 13 | class Meta: 14 | db_table = 'stage' 15 | indexes = [ 16 | models.Index(fields=['name'], name='stage_name_idx') 17 | ] 18 | 19 | name = models.CharField(unique=True, max_length=512) 20 | description = models.TextField(blank=True) 21 | 22 | variables = models.TextField(null=False, help_text="JSON", default="{}", blank=True) 23 | variable_sets = models.ManyToManyField('VariableSet', related_name='stages', blank=True) 24 | 25 | def __str__(self): 26 | return self.name 27 | -------------------------------------------------------------------------------- /vespene/models/variable_set.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018, Michael DeHaan LLC 2 | # License: Apache License Version 2.0 3 | #--------------------------------------------------------------------------- 4 | # variable_set.py - a bucket of reusable variables that may be attached 5 | # to multiple objects, all sharing the same keys and values. 6 | #--------------------------------------------------------------------------- 7 | 8 | from django.contrib.auth.models import Group, User 9 | from django.db import models 10 | 11 | from vespene.models import BaseModel 12 | 13 | 14 | class VariableSet(models.Model, BaseModel): 15 | 16 | class Meta: 17 | db_table = 'variable_sets' 18 | 19 | name = models.CharField(unique=True, max_length=512) 20 | description = models.TextField(blank=True) 21 | variables = models.TextField(help_text="JSON", blank=True, null=False, default="{}") 22 | 23 | created_by = models.ForeignKey(User, related_name='+', null=True, blank=True, on_delete=models.SET_NULL) 24 | owner_groups = models.ManyToManyField(Group, related_name='variable_sets', blank=True) 25 | 26 | def __str__(self): 27 | return self.name 28 | -------------------------------------------------------------------------------- /vespene/models/worker.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018, Michael DeHaan LLC 2 | # License: Apache License Version 2.0 3 | # --------------------------------------------------------------------------- 4 | # worker.py - represents a worker registering current settings and location 5 | # when it starts up. 6 | # -------------------------------------------------------------------------- 7 | 8 | from django.db import models 9 | from vespene.models import BaseModel 10 | 11 | class Worker(models.Model, BaseModel): 12 | 13 | class Meta: 14 | db_table = 'workers' 15 | indexes = [ 16 | models.Index(fields=['worker_uid'], name='worker_uid_idx'), 17 | ] 18 | 19 | worker_uid = models.CharField(unique=True, max_length=512) 20 | hostname = models.CharField(max_length=1024, null=True) 21 | port = models.IntegerField(null=False, default=8080) 22 | build_root = models.CharField(max_length=1024, null=True) 23 | first_checkin = models.DateTimeField(null=True, blank=True) 24 | last_checkin = models.DateTimeField(null=True, blank=True) 25 | fileserving_enabled = models.BooleanField(null=False, default=False) 26 | 27 | def __str__(self): 28 | return self.hostname 29 | -------------------------------------------------------------------------------- /vespene/models/worker_pool.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018, Michael DeHaan LLC 2 | # License: Apache License Version 2.0 3 | # --------------------------------------------------------------------------- 4 | # worker_pool.py - worker pools are named queues with various proprerties. 5 | # each project has one and only one worker pool and individual workers 6 | # farm the queue to build jobs assigned to those queues. More than one 7 | # worker can farm each queue, but a worker pool without any workers is 8 | # non-functional. 9 | # -------------------------------------------------------------------------- 10 | 11 | from django.contrib.auth.models import User 12 | from django.db import models 13 | 14 | from vespene.common.secrets import SecretsManager 15 | from vespene.models import BaseModel 16 | 17 | secrets = SecretsManager() 18 | 19 | class WorkerPool(models.Model, BaseModel): 20 | 21 | class Meta: 22 | db_table = 'worker_pools' 23 | indexes = [ 24 | models.Index(fields=['name'], name='worker_pool_name_idx'), 25 | ] 26 | 27 | name = models.CharField(unique=True, max_length=512) 28 | variables = models.TextField(blank=True, null=True, default="{}") 29 | variable_sets = models.ManyToManyField('VariableSet', related_name='worker_pools', blank=True) 30 | 31 | isolation_method = models.CharField(blank=False, max_length=50) 32 | sudo_user = models.CharField(max_length=100, blank=True, null=True) 33 | permissions_hex = models.CharField(max_length=5, default="0x777", null=False, blank=False, help_text="permissions for build directory") 34 | 35 | sleep_seconds = models.IntegerField(default=10, blank=False, null=False, help_text="how often workers should scan for builds") 36 | auto_abort_minutes = models.IntegerField(default=24*60, blank=False, null=False, help_text="auto-abort queued builds after this amount of time in queue") 37 | build_latest = models.BooleanField(default=True, null=False, help_text="auto-abort duplicate older builds?") 38 | 39 | # autoscaling 40 | autoscaling_enabled = models.BooleanField(default=False, blank=True) 41 | planner = models.CharField(max_length=512, blank=True, null=True) 42 | running_weight = models.FloatField(default=1.0, blank=True, null=False) 43 | queued_weight = models.FloatField(default=1.0, blank=True, null=False) 44 | excess = models.IntegerField(default=0, blank=True, null=False) 45 | multiplier = models.FloatField(default=1.0, blank=True, null=False) 46 | minimum = models.IntegerField(default=0, blank=True, null=False) 47 | maximum = models.IntegerField(default=10, blank=True, null=False) 48 | executor = models.CharField(max_length=512, blank=True, null=True) 49 | executor_command = models.CharField(max_length=1024, blank=True, null=False, default="") 50 | last_autoscaled = models.DateTimeField(null=True, blank=True) 51 | autoscaling_status = models.IntegerField(null=True, blank=True) 52 | reevaluate_minutes = models.IntegerField(null=False, default=10, blank=True) 53 | 54 | # FIXME: the python manage.py cleanup commands do *NOT* currently use these fields. 55 | build_object_shelf_life = models.IntegerField(default=365, blank=False, null=False, help_text="retain build objects for this many days") 56 | build_root_shelf_life = models.IntegerField(default=31, blank=False, null=False, help_text="retain build roots for this many days") 57 | 58 | created_by = models.ForeignKey(User, related_name='+', null=True, blank=True, on_delete=models.SET_NULL) 59 | 60 | def __str__(self): 61 | return self.name 62 | 63 | def save(self, *args, **kwargs): 64 | super().save(*args, **kwargs) 65 | 66 | def as_dict(self): 67 | return dict( 68 | id = self.id, 69 | name = self.name, 70 | variables = self.variables, 71 | isolation_method = self.isolation_method, 72 | sudo_user = self.sudo_user 73 | ) 74 | -------------------------------------------------------------------------------- /vespene/plugins/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018, Michael DeHaan LLC 2 | # License: Apache License Version 2.0 -------------------------------------------------------------------------------- /vespene/plugins/authorization/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018, Michael DeHaan LLC 2 | # License: Apache License Version 2.0 -------------------------------------------------------------------------------- /vespene/plugins/authorization/group_required.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018, Michael DeHaan LLC 2 | # License: Apache License Version 2.0 3 | # ------------------------------------------------------------------------- 4 | # group_required.py - a plugin to limit access to certain types of 5 | # verbs on certain types of nouns to certain groups. See web 'authz' 6 | # documentation for more info. DO NOT USE THIS PLUGIN without 7 | # also enabling the 'ownership' plugin, or the default security 8 | # rules will not work as expected. 9 | # -------------------------------------------------------------------------- 10 | 11 | from django.db.models import Q 12 | from django.core.exceptions import ValidationError 13 | from django.contrib import messages 14 | from vespene.common.logger import Logger 15 | from vespene.manager import Shared 16 | 17 | LOG = Logger() 18 | 19 | class Plugin(object): 20 | 21 | def __init__(self, parameters): 22 | self.cfg = parameters 23 | 24 | def required_groups(self, cls, verb): 25 | name = cls.__name__.split('.')[-1].lower() 26 | class_cfg = self.cfg.get(name, {}) 27 | verb_cfg = class_cfg.get(verb, None) 28 | return verb_cfg 29 | 30 | def matches(self, cls, verb, request): 31 | if request.user.is_superuser: 32 | return True 33 | groups = self.required_groups(cls, verb) 34 | if groups is None: 35 | return True 36 | required = set(self.required_groups(cls, verb)) 37 | groups = set([ g.name for g in request.user.groups.all() ]) 38 | both = required.intersection(groups) 39 | return len(both) > 0 40 | 41 | def check_can_list(self, cls, request, *args, **kwargs): 42 | return self.matches(cls, 'list', request) 43 | 44 | def check_can_create(self, cls, request, *args, **kwargs): 45 | return self.matches(cls, 'create', request) 46 | 47 | def filter_queryset_for_list(self, queryset, request, *args, **kwargs): 48 | return queryset 49 | 50 | def filter_queryset_for_view(self, queryset, request, *args, **kwargs): 51 | return queryset 52 | 53 | def filter_queryset_for_edit(self, queryset, request, *args, **kwargs): 54 | return queryset 55 | 56 | def filter_queryset_for_delete(self, queryset, request, *args, **kwargs): 57 | return queryset 58 | 59 | def check_can_view(self, obj, request, *args, **kwargs): 60 | return self.matches(obj.__class__, 'view', request) 61 | 62 | def check_can_edit(self, obj, request, *args, **kwargs): 63 | return self.matches(obj.__class__, 'edit', request) 64 | 65 | def check_can_delete(self, obj, request, *args, **kwargs): 66 | return self.matches(obj.__class__, 'delete', request) 67 | 68 | def check_can_start(self, obj, request, *args, **kwargs): 69 | rc = self.matches(obj.__class__, 'start', request) 70 | return rc 71 | 72 | def check_can_stop(self, obj, request, *args, **kwargs): 73 | return self.matches(obj.__class__, 'stop', request) 74 | 75 | 76 | -------------------------------------------------------------------------------- /vespene/plugins/autoscale_executor/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018, Michael DeHaan LLC 2 | # License: Apache License Version 2.0 -------------------------------------------------------------------------------- /vespene/plugins/autoscale_executor/shell.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018, Michael DeHaan LLC 2 | # License: Apache License Version 2.0 3 | # ----------------------------------------------------- 4 | # autoscales a worker pool using determined parameters 5 | 6 | import subprocess 7 | import sys 8 | from vespene.common.logger import Logger 9 | from vespene.common.templates import template 10 | 11 | LOG = Logger() 12 | 13 | class Plugin(object): 14 | 15 | def __init__(self): 16 | pass 17 | 18 | def invoke(self, worker_pool, cmd): 19 | with subprocess.Popen(cmd, stdout=subprocess.PIPE, bufsize=1, shell=True, universal_newlines=True) as p: 20 | for line in p.stdout: 21 | LOG.debug("%s | %s" % (worker_pool.name, line)) 22 | if p.returncode != 0: 23 | raise CalledProcessError(p.returncode, p.args) 24 | 25 | def scale_worker_pool(self, worker_pool, parameters): 26 | """ 27 | This plugin just templates out a shell string using Jinja2 and runs it. 28 | See web docs at docs.vespene.io for an example. 29 | """ 30 | 31 | cmd = worker_pool.executor_command 32 | cmd = template(cmd, parameters, strict_undefined=True) 33 | result = self.invoke(worker_pool, cmd) 34 | LOG.debug(result) 35 | 36 | 37 | -------------------------------------------------------------------------------- /vespene/plugins/autoscale_planner/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018, Michael DeHaan LLC 2 | # License: Apache License Version 2.0 -------------------------------------------------------------------------------- /vespene/plugins/autoscale_planner/stock.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018, Michael DeHaan LLC 2 | # License: Apache License Version 2.0 3 | # ----------------------------------------------------- 4 | # calculates how large an autoscaling group for a worker should be 5 | 6 | from django.utils import timezone 7 | from datetime import datetime, timedelta 8 | 9 | from vespene.common.logger import Logger 10 | from vespene.common.templates import template 11 | from vespene.models.build import Build, QUEUED, RUNNING 12 | 13 | import math 14 | import os.path 15 | import json 16 | 17 | LOG = Logger() 18 | 19 | 20 | class Plugin(object): 21 | 22 | def __init__(self): 23 | pass 24 | 25 | def is_time_to_adjust(self, worker_pool): 26 | """ 27 | Determine if it is time to re-evaluate the autoscaling status of this pool 28 | """ 29 | 30 | last = worker_pool.last_autoscaled 31 | if last is None: 32 | return True 33 | now = datetime.now(tz=timezone.utc) 34 | threshold = now - timedelta(minutes=worker_pool.reevaluate_minutes) 35 | result = (worker_pool.last_autoscaled > threshold) 36 | return result 37 | 38 | 39 | def formula(self, worker_pool, x, y): 40 | """ 41 | Apply calculations using weights to return actual desired sizes 42 | """ 43 | 44 | count = x + y 45 | count = math.ceil(count * worker_pool.multiplier) + worker_pool.excess 46 | if count < worker_pool.minimum: 47 | count = worker_pool.minimum 48 | if count > worker_pool.maximum: 49 | count = worker_pool.maximum 50 | return count 51 | 52 | def get_parameters(self, worker_pool): 53 | """ 54 | Get the desired-size parameters for the execution plugin 55 | """ 56 | 57 | running = Build.objects.filter( 58 | status = RUNNING, 59 | worker_pool = worker_pool 60 | ).count() 61 | 62 | if worker_pool.build_latest: 63 | queued = Build.objects.filter( 64 | worker_pool = worker_pool, 65 | status = QUEUED, 66 | ).distinct('project').count() 67 | else: 68 | queued = Build.objects.filter( 69 | worker_pool = worker_pool, 70 | status = QUEUED, 71 | ).count() 72 | 73 | return dict( 74 | worker_pool = worker_pool, 75 | # number of additional worker images that should be spun up 76 | # use this if the provisioning system is NOT declarative 77 | queued_size = self.formula(worker_pool, queued, running), 78 | # total number of worker images that should be spun up total 79 | # use this if the provisioning system IS declarative 80 | size = self.formula(worker_pool, queued, 0), 81 | ) 82 | 83 | 84 | 85 | 86 | 87 | -------------------------------------------------------------------------------- /vespene/plugins/isolation/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018, Michael DeHaan LLC 2 | # License: Apache License Version 2.0 -------------------------------------------------------------------------------- /vespene/plugins/isolation/basic_container.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018, Michael DeHaan LLC 2 | # License: Apache License Version 2.0 3 | # ------------------------------------------------------------------------- 4 | # basic_container.py - this is a docker-cli based isolation strategy 5 | # that runs builds as the result of a "docker build" command, and then 6 | # copies the build directory out of the resultant CLI image. The docker 7 | # containers are short-lived and deleted after the build is done. 8 | # -------------------------------------------------------------------------- 9 | 10 | from vespene.workers import commands 11 | from vespene.common.logger import Logger 12 | from vespene.common.templates import template 13 | 14 | import os.path 15 | import json 16 | 17 | LOG = Logger() 18 | 19 | CONTAINER_TEMPLATE = """ 20 | FROM {{ container_base_image }} 21 | ADD . /tmp/buildroot 22 | RUN chmod +x /tmp/buildroot/vespene_launch.sh 23 | RUN (cd /tmp/buildroot; timeout {{ timeout }} /tmp/buildroot/vespene_launch.sh) 24 | ENTRYPOINT sleep 1000 25 | """ 26 | 27 | class Plugin(object): 28 | 29 | def __init__(self): 30 | pass 31 | 32 | def setup(self, build): 33 | self.build = build 34 | 35 | def _get_script_file_name(self): 36 | # Where should we write the build script? 37 | if not self.build.working_dir: 38 | raise Exception("working directory not set") 39 | return os.path.join(self.build.working_dir, "vespene_launch.sh") 40 | 41 | def _get_json_file_name(self): 42 | return os.path.join(self.build.working_dir, "vespene.json") 43 | 44 | def _get_container_base_image(self): 45 | if not self.build.project.container_base_image: 46 | raise Exception("container base image is not set") 47 | return self.build.project.container_base_image 48 | 49 | def begin(self): 50 | # Write the build script, the chdir into the build root. 51 | self.script_file_name = self._get_script_file_name() 52 | self.json_file_name = self._get_json_file_name() 53 | 54 | fh = open(self.script_file_name, "w") 55 | fh.write(self.build.script) 56 | fh.close() 57 | 58 | fh = open(self.json_file_name, "w") 59 | fh.write(self.build.variables) 60 | fh.close() 61 | 62 | self.prev_dir = os.getcwd() 63 | os.chdir(self.build.working_dir) 64 | 65 | base = self._get_container_base_image() 66 | 67 | self.build.append_message("using container base image: %s" % base) 68 | 69 | context = dict( 70 | timeout = (60 * self.build.project.timeout), 71 | container_base_image = base, 72 | buildroot = self.build.working_dir 73 | ) 74 | contents = template(CONTAINER_TEMPLATE, context) 75 | 76 | fh = open("Dockerfile", "w") 77 | fh.write(contents) 78 | fh.close() 79 | 80 | def end(self): 81 | # Chdir out of the build root 82 | os.chdir(self.prev_dir) 83 | 84 | def execute(self): 85 | 86 | self.build.append_message("----------\nBuilding...") 87 | 88 | img = "vespene_%s" % self.build.id 89 | 90 | cmd = "docker build . -t %s" % img 91 | commands.execute_command(self.build, cmd, log_command=True, output_log=True, message_log=False) 92 | cmd = "docker run -d %s --name %s" % (img, img) 93 | commands.execute_command(self.build, cmd, log_command=True, output_log=True, message_log=False) 94 | cmd = "docker ps -a | grep %s" % img 95 | out = commands.execute_command(self.build, cmd, log_command=True, output_log=True, message_log=True) 96 | token = out.split("\n")[0].split(" ")[0] 97 | cmd = "docker cp %s:/tmp/buildroot ." % (token) 98 | commands.execute_command(self.build, cmd, log_command=True, output_log=False, message_log=True) 99 | cmd = "cp -r buildroot/* ." 100 | commands.execute_command(self.build, cmd, log_command=True, output_log=False, message_log=True) 101 | cmd = "rm -R buildroot" 102 | commands.execute_command(self.build, cmd, log_command=True, output_log=False, message_log=True) 103 | cmd = "docker container stop %s" % (token) 104 | commands.execute_command(self.build, cmd, log_command=True, output_log=False, message_log=True) 105 | cmd = "docker container rm %s" % (token) 106 | commands.execute_command(self.build, cmd, log_command=True, output_log=False, message_log=True) 107 | cmd = "docker image rm %s" % (img) 108 | commands.execute_command(self.build, cmd, log_command=True, output_log=False, message_log=True) 109 | 110 | -------------------------------------------------------------------------------- /vespene/plugins/isolation/sudo.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018, Michael DeHaan LLC 2 | # License: Apache License Version 2.0 3 | # ------------------------------------------------------------------------- 4 | # sudo.py - this is a build isolation strategy that runs sudo to a configured 5 | # username prior to running a build. 6 | # -------------------------------------------------------------------------- 7 | 8 | from vespene.workers import commands 9 | from vespene.common.logger import Logger 10 | 11 | import os.path 12 | import json 13 | import shutil 14 | 15 | LOG = Logger() 16 | 17 | class Plugin(object): 18 | 19 | def __init__(self): 20 | pass 21 | 22 | def setup(self, build): 23 | self.build = build 24 | self.sudo_user = self._get_sudo_user() 25 | self.chmod = self._get_chmod() 26 | 27 | def _get_script_file_name(self): 28 | # Where should we write the build script? 29 | if not self.build.working_dir: 30 | raise Exception("working directory not set") 31 | return os.path.join(self.build.working_dir, "vespene_launch.sh") 32 | 33 | def _get_json_file_name(self): 34 | # Where should we write the build script? 35 | return os.path.join(self.build.working_dir, "vespene.json") 36 | 37 | def _get_sudo_user(self): 38 | # See if a sudo user is configured on the worker pool, if not, just use 'nobody' 39 | user = self.build.project.worker_pool.sudo_user 40 | if not user: 41 | return "nobody" 42 | return user 43 | 44 | def _get_chmod(self): 45 | # Check permissions hex and assert it is sane. 46 | chmod = self.build.project.worker_pool.permissions_hex 47 | if not chmod: 48 | chmod = 0x777 49 | chmod = int(chmod, 16) 50 | return "%X" % chmod 51 | 52 | def begin(self): 53 | 54 | self.build.append_message("running build asunder user: %s" % self.sudo_user) 55 | 56 | # Write the build script, the chdir into the build root. 57 | self.script_file_name = self._get_script_file_name() 58 | self.json_file_name = self._get_json_file_name() 59 | 60 | fh = open(self.script_file_name, "w") 61 | fh.write(self.build.script) 62 | fh.close() 63 | 64 | fh = open(self.json_file_name, "w") 65 | fh.write(self.build.variables) 66 | fh.close() 67 | 68 | self.prev_dir = os.getcwd() 69 | os.chdir(self.build.working_dir) 70 | 71 | def end(self): 72 | # Chdir out of the build root 73 | os.chdir(self.prev_dir) 74 | 75 | def execute(self): 76 | self.build.append_message("----------\nBuilding...") 77 | commands.execute_command(self.build, "chmod -R %s %s" % (self.chmod, self.build.working_dir), output_log=False, message_log=True) 78 | commands.execute_command(self.build, "chmod a+x %s" % self.script_file_name, output_log=False, message_log=True) 79 | if shutil.which('timeout'): 80 | timeout = "timeout %d " % (self.build.project.timeout * 60) 81 | else: 82 | timeout = "" 83 | sudo_command = "sudo -Snk -u %s %s%s" % (self.sudo_user, timeout, self.script_file_name) 84 | self.build.append_message("see 'Output'") 85 | commands.execute_command(self.build, sudo_command, log_command=False, output_log=True, message_log=False) -------------------------------------------------------------------------------- /vespene/plugins/organizations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vespene-io/_old_vespene/c0fbb217dfc5155bf974ab9e06fd43e539da63e0/vespene/plugins/organizations/__init__.py -------------------------------------------------------------------------------- /vespene/plugins/organizations/github.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018, Michael DeHaan LLC 2 | # License: Apache License Version 2.0 3 | # ------------------------------------------------------------------------- 4 | # github.py - plumbing for supporting GitHub within the Vespene 5 | # organizational imports feature 6 | # -------------------------------------------------------------------------- 7 | 8 | import os 9 | import shlex 10 | 11 | from django.db.models import Q 12 | from github import Github 13 | 14 | from vespene.common.logger import Logger 15 | from vespene.workers import commands 16 | 17 | LOG = Logger() 18 | 19 | class Plugin(object): 20 | 21 | def __init__(self, parameters=None): 22 | self.parameters = parameters 23 | if parameters is None: 24 | self.parameters = {} 25 | 26 | def get_handle(self, organization): 27 | scm_login = organization.scm_login 28 | if organization.api_endpoint: 29 | g = Github(scm_login.username, scm_login.password(), base_url=organization.api_endpoint) 30 | else: 31 | g = Github(scm_login.username, scm_login.get_password()) 32 | return g 33 | 34 | def find_all_repos(self, organization, build): 35 | handle = self.get_handle(organization) 36 | org = handle.get_organization(organization.organization_identifier) 37 | repos = org.get_repos(type='all') 38 | results = [] 39 | for repo in repos: 40 | results.append(repo.clone_url) 41 | return results 42 | 43 | def clone_repo(self, organization, build, repo, count): 44 | 45 | # much of code is borrowed from plugins.scm.git - but adapted enough that 46 | # sharing is probably not worthwhile. For instance, this doesn't have 47 | # to deal with SSH checkouts. 48 | 49 | build.append_message("cloning repo...") 50 | 51 | repo = self.fix_scm_url(repo, organization.scm_login.username) 52 | answer_file = commands.answer_file(organization.scm_login.get_password()) 53 | ask_pass = " --config core.askpass=\"%s\"" % answer_file 54 | branch_spec = "--depth 1 --single-branch " 55 | 56 | clone_path = os.path.join(build.working_dir, str(count)) 57 | 58 | try: 59 | # run it 60 | cmd = "git clone %s %s %s %s" % (shlex.quote(repo), clone_path, ask_pass, branch_spec) 61 | output = commands.execute_command(build, cmd, output_log=False, message_log=True) 62 | finally: 63 | # delete the answer file if we had one 64 | os.remove(answer_file) 65 | 66 | return clone_path 67 | 68 | def fix_scm_url(self, repo, username): 69 | 70 | # Adds the username and password into the repo URL before checkout, if possible 71 | # This isn't needed if we are using SSH keys, and that's already handled by SshManager 72 | 73 | for prefix in [ 'https://', 'http://' ]: 74 | if repo.startswith(prefix): 75 | repo = repo.replace(prefix, "") 76 | return "%s%s@%s" % (prefix, username, repo) 77 | return repo 78 | 79 | -------------------------------------------------------------------------------- /vespene/plugins/output/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vespene-io/_old_vespene/c0fbb217dfc5155bf974ab9e06fd43e539da63e0/vespene/plugins/output/__init__.py -------------------------------------------------------------------------------- /vespene/plugins/output/timestamp.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018, Michael DeHaan LLC 2 | # License: Apache License Version 2.0 3 | # ------------------------------------------------------------------------- 4 | # timestamp.py - an output plugin that appends the elapsed time before 5 | # log messages 6 | # -------------------------------------------------------------------------- 7 | 8 | from vespene.workers import commands 9 | from vespene.common.logger import Logger 10 | from datetime import datetime 11 | from django.utils import timezone 12 | 13 | LOG = Logger() 14 | 15 | class Plugin(object): 16 | 17 | def __init__(self, parameters=None): 18 | if parameters is None: 19 | parameters = {} 20 | self.parameters = parameters 21 | self.mode = self.parameters.get('mode', 'elapsed') 22 | if self.mode != 'elapsed': 23 | raise Exception("the only available timestamp mode is 'elapsed'.") 24 | 25 | def filter(self, build, msg): 26 | if msg is None: 27 | return None 28 | now = datetime.now(tz=timezone.utc) 29 | delta = now - build.start_time 30 | minutes = delta.total_seconds() / 60.0 31 | prefix = "%0.2f" % minutes 32 | return "%sm | %s" % (prefix.rjust(8), msg) 33 | -------------------------------------------------------------------------------- /vespene/plugins/scm/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018, Michael DeHaan LLC 2 | # License: Apache License Version 2.0 -------------------------------------------------------------------------------- /vespene/plugins/scm/none.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018, Michael DeHaan LLC 2 | # License: Apache License Version 2.0 3 | # ------------------------------------------------------------------------- 4 | # none.py - this is a SCM module that represents a build that is not 5 | # actually in source control. There would be nothing checked out and 6 | # the build script is just run as an arbitrary script. 7 | # -------------------------------------------------------------------------- 8 | 9 | from vespene.common.logger import Logger 10 | 11 | LOG = Logger() 12 | 13 | class Plugin(object): 14 | 15 | def __init__(self): 16 | pass 17 | 18 | def setup(self, build): 19 | pass 20 | 21 | def get_revision(self): 22 | return "" 23 | 24 | def get_last_commit_user(self): 25 | return "" 26 | 27 | def checkout(self): 28 | return "" 29 | -------------------------------------------------------------------------------- /vespene/plugins/scm/svn.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018, Michael DeHaan LLC 2 | # License: Apache License Version 2.0 3 | # ------------------------------------------------------------------------- 4 | # svn.py - this code contains support for the old-and-busted Subversion 5 | # version control system, for those that are not yet using git, which 6 | # is the new hotness. It currently assumes repos are publically accessible 7 | # and because SVN doesn't really have "real" branches, ignores the 8 | # branch parameter. Upgrades from users of SVN setups are quite welcome as 9 | # are additions of other SCM types. 10 | # -------------------------------------------------------------------------- 11 | 12 | import shlex 13 | 14 | from vespene.common.logger import Logger 15 | from vespene.workers import commands 16 | 17 | LOG = Logger() 18 | 19 | class Plugin(object): 20 | 21 | def __init__(self): 22 | pass 23 | 24 | def setup(self, build): 25 | self.build = build 26 | self.project = build.project 27 | self.repo = build.project.repo_url 28 | 29 | def info_extract(self, attribute): 30 | cmd = "(cd %s; svn info | grep \"%s\")" % (self.build.working_dir, attribute) 31 | out = commands.execute_command(self.build, cmd, output_log=False, message_log=True) 32 | if ":" in out: 33 | return out.split(":")[-1].strip() 34 | return None 35 | 36 | def get_revision(self): 37 | return self.info_extract("Last Changed Rev:") 38 | 39 | def get_last_commit_user(self): 40 | return self.info_extract("Last Changed Author:") 41 | 42 | def checkout(self): 43 | self.build.append_message("----------\nCloning repository...") 44 | cmd = "svn checkout --non-interactive --quiet %s %s" % (shlex.quote(self.repo), self.build.working_dir) 45 | return commands.execute_command(self.build, cmd, output_log=False, message_log=True) 46 | -------------------------------------------------------------------------------- /vespene/plugins/secrets/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vespene-io/_old_vespene/c0fbb217dfc5155bf974ab9e06fd43e539da63e0/vespene/plugins/secrets/__init__.py -------------------------------------------------------------------------------- /vespene/plugins/secrets/basic.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018, Michael DeHaan LLC 2 | # License: Apache License Version 2.0 3 | # ------------------------------------------------------------------------- 4 | # basic.py - this is a cloaking plugin for symetric encryption of secrets. 5 | # it's not intended to be a fortress but for the implementation to use 6 | # when you're not using something more complex. It is versioned by implementation 7 | # classes so the plugin can support improvements in existing cloaked secrets 8 | # when required. 9 | # -------------------------------------------------------------------------- 10 | 11 | from django.conf import settings 12 | from cryptography import fernet 13 | import binascii 14 | from vespene.common.logger import Logger 15 | 16 | LOG = Logger() 17 | 18 | class BasicV1(object): 19 | 20 | HEADER = "[VESPENE-CLOAK][BASIC][V1]" 21 | 22 | def __init__(self): 23 | pass 24 | 25 | def cloak(self, msg): 26 | symetric = settings.SYMETRIC_SECRET_KEY 27 | ff = fernet.Fernet(symetric) 28 | msg = msg.encode('utf-8') 29 | enc = ff.encrypt(msg) 30 | henc = binascii.hexlify(enc).decode('utf-8') 31 | return "%s%s" % (self.HEADER, henc) 32 | 33 | def decloak(self, msg): 34 | symetric = settings.SYMETRIC_SECRET_KEY 35 | ff = fernet.Fernet(symetric) 36 | henc = msg.replace(self.HEADER, "", 1) 37 | enc = binascii.unhexlify(henc) 38 | msg = ff.decrypt(enc) 39 | rc = msg.decode('utf-8') 40 | return rc 41 | 42 | class Plugin(object): 43 | 44 | HEADER = "[VESPENE-CLOAK][BASIC]" 45 | 46 | def __init__(self): 47 | pass 48 | 49 | def implementation_for_version(self, msg): 50 | if msg.startswith(BasicV1.HEADER): 51 | return BasicV1() 52 | raise Exception("unknown cloaking version") 53 | 54 | def cloak(self, msg): 55 | return BasicV1().cloak(msg) 56 | 57 | def decloak(self, msg): 58 | impl = self.implementation_for_version(msg) 59 | return impl.decloak(msg) 60 | 61 | def recognizes(self, msg): 62 | if settings.SYMETRIC_SECRET_KEY is None: 63 | # user didn't run 'make secrets' yet, so disable the plugin 64 | return False 65 | return msg.startswith(self.HEADER) -------------------------------------------------------------------------------- /vespene/plugins/triggers/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018, Michael DeHaan LLC 2 | # License: Apache License Version 2.0 -------------------------------------------------------------------------------- /vespene/plugins/triggers/command.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018, Michael DeHaan LLC 2 | # License: Apache License Version 2.0 3 | # ------------------------------------------------------------------------- 4 | # command.py - this is a plugin that runs an arbitrary CLI command when 5 | # a trigger event occurs. Read the 'triggers' web documentation for 6 | # examples. This command will run the command string through Jinja2 prior 7 | # to execution, allowing for substituting in the build path, build number, 8 | # project name, and all kinds of other things. For instance {{ build.working_dir }} 9 | # and {{ project.name }} are things you could use in the command line. 10 | # -------------------------------------------------------------------------- 11 | 12 | from vespene.workers import commands 13 | from vespene.common.templates import template 14 | 15 | class Plugin(object): 16 | 17 | def __init__(self, args): 18 | self.args = args 19 | 20 | def execute_hook(self, build, context): 21 | 22 | command = self.args 23 | command = template(command, context) 24 | commands.execute_command(build, command, log_command=True, output_log=False, message_log=True) 25 | -------------------------------------------------------------------------------- /vespene/plugins/triggers/slack.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018, Michael DeHaan LLC 2 | # License: Apache License Version 2.0 3 | # ------------------------------------------------------------------------- 4 | # slack.py - this is a plugin that sends slack notifications when builds 5 | # start or finish. It could easily be copied to add support for HipChat 6 | # or IRC, and contributions of those modules would be fantastic. Extensions 7 | # to move the configuration strings out of this file, and support more features 8 | # of Slack are also welcome. 9 | # -------------------------------------------------------------------------- 10 | 11 | from slackclient import SlackClient 12 | 13 | from vespene.common.logger import Logger 14 | from vespene.models.build import SUCCESS 15 | from vespene.common.templates import template 16 | 17 | LOG = Logger() 18 | 19 | PRE_TEMPLATE = "Vespene Build {{ build.id }} for project \"{{ build.project.name }}\" started" 20 | SUCCESS_TEMPLATE = "Vespene Build {{ build.id }} for project \"{{ build.project.name }}\" succeeded" 21 | FAILURE_TEMPLATE = "Vespene Build {{ build.id }} for project \"{{ build.project.name }}\" failed" 22 | 23 | class Plugin(object): 24 | 25 | def __init__(self, args): 26 | self.params = args 27 | self.channel = self.params['channel'] 28 | self.token = self.params['token'] 29 | self.client = SlackClient(self.token) 30 | 31 | def execute_hook(self, build, context): 32 | 33 | LOG.debug("executing slack hook") 34 | 35 | if context['hook'] == 'pre': 36 | slack_template = PRE_TEMPLATE 37 | elif build.status == SUCCESS: 38 | slack_template = SUCCESS_TEMPLATE 39 | else: 40 | slack_template = FAILURE_TEMPLATE 41 | 42 | msg = template(slack_template, context, strict_undefined=True) 43 | 44 | self.client.api_call("chat.postMessage", channel=self.channel, text=msg) 45 | -------------------------------------------------------------------------------- /vespene/plugins/variables/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018, Michael DeHaan LLC 2 | # License: Apache License Version 2.0 3 | -------------------------------------------------------------------------------- /vespene/plugins/variables/common.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018, Michael DeHaan LLC 2 | # License: Apache License Version 2.0 3 | # ------------------------------------------------------------------------- 4 | # common.py - the way build scripts are templated (and also vespene.json in 5 | # the build root) is templated is defined by plugins like these. 'common' 6 | # contains the basic behavior of sourcing in all the fields in Vespene 7 | # that are explicitly about variables. This is the 'make the web UI variable 8 | # fields work' behavior, more or less. 9 | # -------------------------------------------------------------------------- 10 | 11 | import json 12 | import traceback 13 | 14 | class Plugin(object): 15 | 16 | def __init__(self): 17 | pass 18 | 19 | def _get_variables(self, obj): 20 | 21 | """ 22 | For a given object, get all the variables in attached variable sets 23 | and variables together. 24 | """ 25 | 26 | variables = dict() 27 | 28 | for x in obj.variable_sets.all(): 29 | try: 30 | set_variables = json.loads(x.variables) 31 | except: 32 | #traceback.print_exc() 33 | raise Exception("failed to parse JSON from Variable Set (%s): %s" % (x.name, x.variables)) 34 | 35 | if type(set_variables) == dict: 36 | variables.update(set_variables) 37 | 38 | try: 39 | obj_variables = json.loads(obj.variables) 40 | except: 41 | raise Exception("failed to parse JSON from object: %s" % obj.variables) 42 | 43 | 44 | if type(obj_variables) == dict: 45 | variables.update(obj_variables) 46 | 47 | return variables 48 | 49 | def compute(self, project, existing_variables): 50 | 51 | variables = dict() 52 | variables.update(self._get_variables(project.worker_pool)) 53 | 54 | if project.pipeline: 55 | variables.update(self._get_variables(project.pipeline)) 56 | 57 | if project.stage: 58 | variables.update(self._get_variables(project.stage)) 59 | 60 | variables.update(self._get_variables(project)) 61 | 62 | return variables 63 | -------------------------------------------------------------------------------- /vespene/plugins/variables/pipelines.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018, Michael DeHaan LLC 2 | # License: Apache License Version 2.0 3 | # ------------------------------------------------------------------------- 4 | # pipelines.py - this logic allows the variables output (see web pipeline docs) in 5 | # the build textual output to be used as input variables to future pipeline 6 | # steps. It's useful if using pipelines and wanting to do something like making 7 | # a deployment step aware of an AWS image ID that got built in a previous step. 8 | # -------------------------------------------------------------------------- 9 | 10 | import json 11 | from vespene.models.project import Project 12 | 13 | class Plugin(object): 14 | 15 | 16 | def __init__(self): 17 | pass 18 | 19 | def compute(self, project, existing_variables): 20 | """ 21 | Get all the variables to pass to Jinja2 for a given project, including snippets. 22 | (This is the variable precedence algorithm, right here). 23 | """ 24 | 25 | output_variables = dict() 26 | if project.pipeline and project.stage: 27 | stage = project.stage 28 | for previous_stage in project.pipeline.all_previous_stages(stage): 29 | previous_projects = Project.objects.filter(pipeline=project.pipeline, stage=previous_stage).all() 30 | for previous in previous_projects: 31 | last = previous.last_successful_build 32 | if not last: 33 | continue 34 | try: 35 | variables = json.loads(last.output_variables) 36 | except: 37 | variables = dict() 38 | if variables: 39 | output_variables.update(variables) 40 | return output_variables 41 | -------------------------------------------------------------------------------- /vespene/plugins/variables/snippets.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018, Michael DeHaan LLC 2 | # License: Apache License Version 2.0 3 | # ------------------------------------------------------------------------- 4 | # snippets.py - this rather simple plugin makes snippets available to 5 | # Jinja2 as variables. Snippet names containing "-" or " " aren't valid Python 6 | # variable so we replace those characters. We may need to make this replacement 7 | # smarter in the future, but it is a start. 8 | # -------------------------------------------------------------------------- 9 | 10 | from vespene.models.snippet import Snippet 11 | from vespene.common.templates import template 12 | 13 | class Plugin(object): 14 | 15 | def __init__(self): 16 | pass 17 | 18 | def compute(self, project, existing_variables): 19 | """ 20 | We have to Jinja2-evaluate all the snippets before injecting them into 21 | the dictionary for the build script evaluation. 22 | """ 23 | 24 | results = dict() 25 | for x in Snippet.objects.order_by('name').all(): 26 | # FIXME: on error, load something into the template an empty string and log it 27 | name = x.name.replace("-","_").replace(" ","_") 28 | results[name] = template(x.text, existing_variables, strict_undefined=False) 29 | return results 30 | -------------------------------------------------------------------------------- /vespene/settings.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018, Michael DeHaan LLC 2 | # License: Apache License Version 2.0 3 | # ------------------------------------------------------------------------- 4 | # settings.py - the central Django settings file. Rather than edit this 5 | # use files in /etc/vespene/settings.d/, though we'll also accept 6 | # some other locations. 7 | # -------------------------------------------------------------------------- 8 | 9 | from split_settings.tools import include, optional 10 | 11 | include( 12 | 'config/*.py', 13 | optional('local_settings.py'), 14 | optional('/etc/vespene/settings.py'), 15 | optional('/etc/vespene/settings.d/*.py') 16 | ) 17 | -------------------------------------------------------------------------------- /vespene/static/css/regular.css: -------------------------------------------------------------------------------- 1 | /*! 2 | * Font Awesome Free 5.1.1 by @fontawesome - https://fontawesome.com 3 | * License - https://fontawesome.com/license (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) 4 | */ 5 | @font-face { 6 | font-family: 'Font Awesome 5 Free'; 7 | font-style: normal; 8 | font-weight: 400; 9 | src: url("../webfonts/fa-regular-400.eot"); 10 | src: url("../webfonts/fa-regular-400.eot?#iefix") format("embedded-opentype"), url("../webfonts/fa-regular-400.woff2") format("woff2"), url("../webfonts/fa-regular-400.woff") format("woff"), url("../webfonts/fa-regular-400.ttf") format("truetype"), url("../webfonts/fa-regular-400.svg#fontawesome") format("svg"); } 11 | 12 | .far { 13 | font-family: 'Font Awesome 5 Free'; 14 | font-weight: 400; } 15 | -------------------------------------------------------------------------------- /vespene/static/css/regular.min.css: -------------------------------------------------------------------------------- 1 | /*! 2 | * Font Awesome Free 5.1.1 by @fontawesome - https://fontawesome.com 3 | * License - https://fontawesome.com/license (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) 4 | */ 5 | @font-face{font-family:"Font Awesome 5 Free";font-style:normal;font-weight:400;src:url(../webfonts/fa-regular-400.eot);src:url(../webfonts/fa-regular-400.eot?#iefix) format("embedded-opentype"),url(../webfonts/fa-regular-400.woff2) format("woff2"),url(../webfonts/fa-regular-400.woff) format("woff"),url(../webfonts/fa-regular-400.ttf) format("truetype"),url(../webfonts/fa-regular-400.svg#fontawesome) format("svg")}.far{font-family:"Font Awesome 5 Free";font-weight:400} -------------------------------------------------------------------------------- /vespene/static/css/solid.css: -------------------------------------------------------------------------------- 1 | /*! 2 | * Font Awesome Free 5.1.1 by @fontawesome - https://fontawesome.com 3 | * License - https://fontawesome.com/license (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) 4 | */ 5 | @font-face { 6 | font-family: 'Font Awesome 5 Free'; 7 | font-style: normal; 8 | font-weight: 900; 9 | src: url("../webfonts/fa-solid-900.eot"); 10 | src: url("../webfonts/fa-solid-900.eot?#iefix") format("embedded-opentype"), url("../webfonts/fa-solid-900.woff2") format("woff2"), url("../webfonts/fa-solid-900.woff") format("woff"), url("../webfonts/fa-solid-900.ttf") format("truetype"), url("../webfonts/fa-solid-900.svg#fontawesome") format("svg"); } 11 | 12 | .fa, 13 | .fas { 14 | font-family: 'Font Awesome 5 Free'; 15 | font-weight: 900; } 16 | -------------------------------------------------------------------------------- /vespene/static/css/solid.min.css: -------------------------------------------------------------------------------- 1 | /*! 2 | * Font Awesome Free 5.1.1 by @fontawesome - https://fontawesome.com 3 | * License - https://fontawesome.com/license (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) 4 | */ 5 | @font-face{font-family:"Font Awesome 5 Free";font-style:normal;font-weight:900;src:url(../webfonts/fa-solid-900.eot);src:url(../webfonts/fa-solid-900.eot?#iefix) format("embedded-opentype"),url(../webfonts/fa-solid-900.woff2) format("woff2"),url(../webfonts/fa-solid-900.woff) format("woff"),url(../webfonts/fa-solid-900.ttf) format("truetype"),url(../webfonts/fa-solid-900.svg#fontawesome) format("svg")}.fa,.fas{font-family:"Font Awesome 5 Free";font-weight:900} -------------------------------------------------------------------------------- /vespene/static/css/vespene.css: -------------------------------------------------------------------------------- 1 | #universe { 2 | margin: auto; 3 | margin-left: 20px; 4 | margin-right: 20px; 5 | margin-top: 20px; 6 | margin-bottom: 20px; 7 | } 8 | 9 | #content { 10 | margin: auto; 11 | margin-top: 10px; 12 | } 13 | 14 | #pagination { 15 | margin-top: 10px; 16 | display: inline-block; 17 | } 18 | 19 | #actions { 20 | margin-top: 10px; 21 | width: 70%; 22 | display: inline-block; 23 | float: right; 24 | } 25 | 26 | #vespene_nav { 27 | margin-top: 17px; 28 | } 29 | 30 | fieldset[disabled] { 31 | pointer-events: none; 32 | opacity: .65; 33 | } 34 | 35 | -------------------------------------------------------------------------------- /vespene/static/js/vespene.js: -------------------------------------------------------------------------------- 1 | var csrf_token = '{{ csrf_token }}'; 2 | 3 | function is_valid_json(data) { 4 | try { 5 | response = jQuery.parseJSON(data); 6 | return true; 7 | } catch { 8 | return false; 9 | } 10 | } 11 | 12 | function no_validate_form_hack() { 13 | $("#editForm").attr('novalidate', 'novalidate'); 14 | } 15 | 16 | function do_post(url, csrf) { 17 | data = { 18 | 'csrfmiddlewaretoken' : csrf 19 | }; 20 | var req = $.post(url, data, function() { 21 | }) 22 | .done(function() { 23 | location.reload() 24 | }) 25 | .fail(function(err) { 26 | alert(err.responseText); 27 | }) 28 | .always(function() { 29 | }); 30 | } 31 | 32 | function confirm_post(url, prompt, csrf) { 33 | var answer = confirm(prompt) 34 | if (answer) { 35 | do_post(url, csrf); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /vespene/static/png/vespene_logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vespene-io/_old_vespene/c0fbb217dfc5155bf974ab9e06fd43e539da63e0/vespene/static/png/vespene_logo.png -------------------------------------------------------------------------------- /vespene/static/webfonts/fa-brands-400.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vespene-io/_old_vespene/c0fbb217dfc5155bf974ab9e06fd43e539da63e0/vespene/static/webfonts/fa-brands-400.eot -------------------------------------------------------------------------------- /vespene/static/webfonts/fa-brands-400.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vespene-io/_old_vespene/c0fbb217dfc5155bf974ab9e06fd43e539da63e0/vespene/static/webfonts/fa-brands-400.ttf -------------------------------------------------------------------------------- /vespene/static/webfonts/fa-brands-400.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vespene-io/_old_vespene/c0fbb217dfc5155bf974ab9e06fd43e539da63e0/vespene/static/webfonts/fa-brands-400.woff -------------------------------------------------------------------------------- /vespene/static/webfonts/fa-brands-400.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vespene-io/_old_vespene/c0fbb217dfc5155bf974ab9e06fd43e539da63e0/vespene/static/webfonts/fa-brands-400.woff2 -------------------------------------------------------------------------------- /vespene/static/webfonts/fa-regular-400.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vespene-io/_old_vespene/c0fbb217dfc5155bf974ab9e06fd43e539da63e0/vespene/static/webfonts/fa-regular-400.eot -------------------------------------------------------------------------------- /vespene/static/webfonts/fa-regular-400.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vespene-io/_old_vespene/c0fbb217dfc5155bf974ab9e06fd43e539da63e0/vespene/static/webfonts/fa-regular-400.ttf -------------------------------------------------------------------------------- /vespene/static/webfonts/fa-regular-400.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vespene-io/_old_vespene/c0fbb217dfc5155bf974ab9e06fd43e539da63e0/vespene/static/webfonts/fa-regular-400.woff -------------------------------------------------------------------------------- /vespene/static/webfonts/fa-regular-400.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vespene-io/_old_vespene/c0fbb217dfc5155bf974ab9e06fd43e539da63e0/vespene/static/webfonts/fa-regular-400.woff2 -------------------------------------------------------------------------------- /vespene/static/webfonts/fa-solid-900.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vespene-io/_old_vespene/c0fbb217dfc5155bf974ab9e06fd43e539da63e0/vespene/static/webfonts/fa-solid-900.eot -------------------------------------------------------------------------------- /vespene/static/webfonts/fa-solid-900.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vespene-io/_old_vespene/c0fbb217dfc5155bf974ab9e06fd43e539da63e0/vespene/static/webfonts/fa-solid-900.ttf -------------------------------------------------------------------------------- /vespene/static/webfonts/fa-solid-900.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vespene-io/_old_vespene/c0fbb217dfc5155bf974ab9e06fd43e539da63e0/vespene/static/webfonts/fa-solid-900.woff -------------------------------------------------------------------------------- /vespene/static/webfonts/fa-solid-900.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vespene-io/_old_vespene/c0fbb217dfc5155bf974ab9e06fd43e539da63e0/vespene/static/webfonts/fa-solid-900.woff2 -------------------------------------------------------------------------------- /vespene/version.py: -------------------------------------------------------------------------------- 1 | VERSION = "1.0-dev" 2 | -------------------------------------------------------------------------------- /vespene/views/fileserving.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018, Michael DeHaan LLC 2 | # License: Apache License Version 2.0 3 | # ------------------------------------------------------------------------- 4 | # fileserving.py - serve up build roots from different workers 5 | # -------------------------------------------------------------------------- 6 | 7 | import traceback 8 | from urllib.parse import parse_qs 9 | import os 10 | from wsgiref.util import FileWrapper 11 | import mimetypes 12 | 13 | from django.shortcuts import get_object_or_404, render 14 | from django.http import HttpResponse, HttpResponseServerError 15 | from django.http import HttpResponse 16 | from django.views.decorators.csrf import csrf_exempt 17 | from django.conf import settings 18 | from django.core.exceptions import PermissionDenied 19 | 20 | from vespene.models.build import Build 21 | from vespene.common.logger import Logger 22 | 23 | LOG = Logger() 24 | 25 | def legal_path(path): 26 | segments = os.path.split(path) 27 | if "." in segments or ".." in segments: 28 | return False 29 | 30 | for x in range(1, len(segments)): 31 | rejoined = os.pathsep.join(segments[0:x]) 32 | if os.path.islink(rejoined): 33 | return False 34 | return True 35 | 36 | def do_serve_file(build, request, root, path): 37 | fh = open(path, 'rb') 38 | wrapper = FileWrapper(fh) 39 | mtype = mimetypes.guess_type(path) 40 | response = HttpResponse(wrapper, content_type=mtype) 41 | response['Content-Length'] = os.path.getsize(path) 42 | return response 43 | 44 | def do_serve_dir(build, request, root, path): 45 | contents = os.listdir(path) 46 | contents = [ c for c in contents if contents not in [ ".", ".."]] 47 | files = [] 48 | for c in contents: 49 | joined = os.path.join(path, c) 50 | partial = joined.replace(root, "", 1) 51 | item = dict() 52 | item["directory"] = os.path.isdir(joined) 53 | item["full_path"] = partial 54 | item["basename"] = c 55 | # TODO: add size, time 56 | files.append(item) 57 | context = dict( 58 | path = path, 59 | url = "%s/%s" % (settings.FILESERVING_URL, build.id), 60 | files = files 61 | ) 62 | return render(request, "dir_index.j2", context=context) 63 | 64 | def get_path(request): 65 | qs = parse_qs(request.META['QUERY_STRING']) 66 | path = qs.get('p',None) 67 | if path is None: 68 | path = "/" 69 | else: 70 | path = path[0] 71 | return path 72 | 73 | @csrf_exempt 74 | def serve_file(request, *args, **kwargs): 75 | path = get_path(request) 76 | return fileserver(request, path, *args, **kwargs) 77 | 78 | @csrf_exempt 79 | def serve_dir(request, *args, **kwargs): 80 | path = get_path(request) 81 | return fileserver(request, path, *args, **kwargs) 82 | 83 | def fileserver(request, fname, *args, **kwargs): 84 | """ 85 | Handle incoming file serving requests for a worker to serve 86 | up it's own build roots. Doesn't handle requests to other 87 | workers. 88 | """ 89 | if not settings.FILESERVING_ENABLED: 90 | raise PermissionDenied() 91 | 92 | build = get_object_or_404(Build.objects, pk=kwargs.get('pk')) 93 | worker = build.worker 94 | if worker is None: 95 | raise PermissionDenied() 96 | 97 | root = os.path.join(settings.BUILD_ROOT, str(build.pk)) 98 | if not os.path.exists(root): 99 | raise PermissionDenied() 100 | 101 | while fname.startswith("/"): 102 | fname = fname.replace("/","",1) 103 | fname = os.path.join(root, fname) 104 | if not legal_path(fname): 105 | raise PermissionDenied() 106 | if os.path.isdir(fname): 107 | return do_serve_dir(build, request, root, fname) 108 | else: 109 | return do_serve_file(build, request, root, fname) 110 | -------------------------------------------------------------------------------- /vespene/views/group.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018, Michael DeHaan LLC 2 | # License: Apache License Version 2.0 3 | 4 | from vespene.views import forms 5 | from django.contrib.auth.models import Group 6 | from vespene.views import BaseView 7 | from django.conf import settings 8 | from django.core.exceptions import PermissionDenied 9 | 10 | class GroupView(BaseView): 11 | model = Group 12 | form = forms.GroupForm 13 | view_prefix = 'group' 14 | object_label = 'Group' 15 | supports_new = True 16 | supports_edit = True 17 | supports_delete = True 18 | 19 | @classmethod 20 | def get_queryset(cls, request): 21 | if settings.ALLOW_UI_GROUP_CREATION: 22 | return Group.objects 23 | else: 24 | raise PermissionDenied 25 | 26 | #@classmethod 27 | #def description_column(cls, obj): 28 | # return obj.description 29 | 30 | GroupView.extra_columns = [ 31 | #('Description', SnippetView.description_column) 32 | ] 33 | -------------------------------------------------------------------------------- /vespene/views/organization.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018, Michael DeHaan LLC 2 | # License: Apache License Version 2.0 3 | 4 | import json 5 | import traceback 6 | from django.shortcuts import get_object_or_404, redirect 7 | from django.http.response import HttpResponse, HttpResponseServerError 8 | from vespene.views import forms 9 | from vespene.models.organization import Organization 10 | from vespene.manager.permissions import PermissionsManager 11 | from vespene.manager.jobkick import start_project_from_ui 12 | from vespene.views.view_helpers import (build_status_icon, get_context, icon, link, project_controls_icon, template) 13 | from vespene.views import BaseView 14 | 15 | permissions = PermissionsManager() 16 | 17 | class OrganizationView(BaseView): 18 | """ 19 | View configuration for the Project model. 20 | Projects are fairly editable, and also offer stop/stop navigation for builds. 21 | """ 22 | 23 | model = Organization 24 | form = forms.OrganizationForm 25 | view_prefix = 'organization' 26 | object_label = 'Organization' 27 | supports_new = True 28 | supports_edit = True 29 | supports_delete = True 30 | 31 | @classmethod 32 | def get_queryset(cls, request): 33 | return Organization.objects.prefetch_related('last_build', 'active_build', 'last_successful_build') 34 | 35 | @classmethod 36 | def last_build_column(cls, obj): 37 | return build_status_icon(obj.last_build, include_buildroot_link=False) 38 | 39 | @classmethod 40 | def active_build_column(cls, obj): 41 | return build_status_icon(obj.last_build, include_buildroot_link=False) 42 | 43 | @classmethod 44 | def last_successful_build_column(cls, obj): 45 | return build_status_icon(obj.last_successful_build, include_buildroot_link=False) 46 | 47 | OrganizationView.extra_columns = [ 48 | ('Active Import', OrganizationView.active_build_column), 49 | ('Last Import', OrganizationView.last_build_column), 50 | ('Last Successful Import', OrganizationView.last_successful_build_column), 51 | ] -------------------------------------------------------------------------------- /vespene/views/pipeline.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018, Michael DeHaan LLC 2 | # License: Apache License Version 2.0 3 | 4 | from django.shortcuts import get_object_or_404 5 | from vespene.views import forms 6 | from django.http.response import HttpResponse, HttpResponseServerError 7 | from vespene.common.logger import Logger 8 | from vespene.models.pipeline import Pipeline 9 | from vespene.models.project import Project 10 | from vespene.manager.permissions import PermissionsManager 11 | from vespene.views.view_helpers import (build_status_icon, 12 | format_time, icon, link, 13 | project_controls_icon, template) 14 | from vespene.views import BaseView 15 | 16 | LOG = Logger() 17 | 18 | permissions = PermissionsManager() 19 | 20 | class PipelineView(BaseView): 21 | model = Pipeline 22 | form = forms.PipelineForm 23 | view_prefix = 'pipeline' 24 | object_label = 'Pipeline' 25 | supports_new = True 26 | supports_edit = True 27 | supports_delete = True 28 | 29 | @classmethod 30 | def get_queryset(cls, request): 31 | return Pipeline.objects 32 | 33 | @classmethod 34 | def map(cls, request, *args, **kwargs): 35 | """ 36 | This generates a custom web page that shows all the projects that are in each pipeline. 37 | The view is a grid, with each stage presented in order of appearance. 38 | """ 39 | 40 | pipeline = get_object_or_404(Pipeline, pk=kwargs.get('pk')) 41 | 42 | def describe_project(project): 43 | """ 44 | This draws a cell within the table for each project that is a member of each stage. 45 | """ 46 | if project is None: 47 | return "" 48 | if project.active_build is None: 49 | which = build_status_icon(project.last_build, compact=False) 50 | else: 51 | which = build_status_icon(project.active_build, compact=False) 52 | controls = project_controls_icon(project, project.active_build, compact=True) 53 | project_link = link("/ui/projects/%d/detail" % project.id, project.name) 54 | return "%s
%s %s" % (project_link, which, controls) 55 | 56 | stages = pipeline.all_stages() 57 | projects_by_stage = dict() 58 | max_width = 0 59 | # Build up a hash table of each project in each stage. 60 | # TODO: move this code into models/stage.py 61 | for stage in stages: 62 | projects = Project.objects.filter(stage=stage, pipeline=pipeline).order_by('name').all() 63 | width = len(projects) 64 | if width > max_width: 65 | max_width = width 66 | projects_by_stage[stage.name] = projects 67 | 68 | # This pads out the tables to make the table display even, because not every 69 | # stage may have the same number of rows. 70 | for (k,v) in projects_by_stage.items(): 71 | stage_length = len(v) 72 | values = [ x for x in v ] 73 | if stage_length < max_width: 74 | padding = max_width - stage_length 75 | values.extend(None for x in range(0,padding)) 76 | projects_by_stage[k] = values 77 | 78 | # With all data calculated, render the template. 79 | context = dict( 80 | pipeline = pipeline, 81 | width = max_width, 82 | stages = stages, 83 | projects_by_stage = projects_by_stage, 84 | describe_project = describe_project 85 | ) 86 | return template(request, 'pipeline_map.j2', context) 87 | 88 | @classmethod 89 | def status_column(cls, obj): 90 | return obj.explain_status() 91 | 92 | @classmethod 93 | def map_column(cls, obj): 94 | # show a link to see the build history of this project 95 | my_link = "/ui/pipelines/%s/map" % obj.id 96 | my_icon = icon('fa-map', family='fas', tooltip='Map View') 97 | return link(my_link, my_icon) 98 | 99 | @classmethod 100 | def last_completed_date_column(cls, obj): 101 | return format_time(obj.last_completed_date) 102 | 103 | @classmethod 104 | def last_completed_by_column(cls, obj): 105 | if obj.last_completed_by: 106 | return build_status_icon(obj.last_completed_by) 107 | else: 108 | return "-" 109 | 110 | PipelineView.extra_columns = [ 111 | ('Last Completed Date', PipelineView.last_completed_date_column), 112 | ('Last Endpoint Build', PipelineView.last_completed_by_column), 113 | ('Map', PipelineView.map_column) 114 | ] -------------------------------------------------------------------------------- /vespene/views/service_login.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018, Michael DeHaan LLC 2 | # License: Apache License Version 2.0 3 | 4 | 5 | from vespene.views import forms 6 | from vespene.models.service_login import ServiceLogin 7 | from vespene.views import BaseView 8 | 9 | class ServiceLoginView(BaseView): 10 | model = ServiceLogin 11 | form = forms.ServiceLoginForm 12 | view_prefix = 'service_login' 13 | object_label = 'Service Login' 14 | supports_new = True 15 | supports_edit = True 16 | supports_delete = True 17 | 18 | @classmethod 19 | def get_queryset(cls, request): 20 | return ServiceLogin.objects 21 | 22 | ServiceLoginView.extra_columns = [] -------------------------------------------------------------------------------- /vespene/views/snippet.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018, Michael DeHaan LLC 2 | # License: Apache License Version 2.0 3 | 4 | from vespene.views import forms 5 | from vespene.models.snippet import Snippet 6 | from vespene.views import BaseView 7 | import html 8 | 9 | class SnippetView(BaseView): 10 | model = Snippet 11 | form = forms.SnippetForm 12 | view_prefix = 'snippet' 13 | object_label = 'Snippet' 14 | supports_new = True 15 | supports_edit = True 16 | supports_delete = True 17 | 18 | @classmethod 19 | def get_queryset(cls, request): 20 | return Snippet.objects 21 | 22 | @classmethod 23 | def description_column(cls, obj): 24 | return html.escape(obj.description) 25 | 26 | SnippetView.extra_columns = [ 27 | ('Description', SnippetView.description_column) 28 | ] -------------------------------------------------------------------------------- /vespene/views/ssh_key.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018, Michael DeHaan LLC 2 | # License: Apache License Version 2.0 3 | 4 | from vespene.views import forms 5 | from vespene.models.ssh_key import SshKey 6 | from vespene.views import BaseView 7 | 8 | class SshKeyView(BaseView): 9 | model = SshKey 10 | form = forms.SshKeyForm 11 | view_prefix = 'ssh_key' 12 | object_label = 'SSH Key' 13 | supports_new = True 14 | supports_edit = True 15 | supports_delete = True 16 | 17 | @classmethod 18 | def get_queryset(cls, request): 19 | return SshKey.objects 20 | 21 | SshKeyView.extra_columns = [] -------------------------------------------------------------------------------- /vespene/views/stage.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018, Michael DeHaan LLC 2 | # License: Apache License Version 2.0 3 | 4 | from vespene.views import forms 5 | from vespene.models.stage import Stage 6 | from vespene.views import BaseView 7 | 8 | class StageView(BaseView): 9 | model = Stage 10 | form = forms.StageForm 11 | view_prefix = 'stage' 12 | object_label = 'Stage' 13 | supports_new = True 14 | supports_edit = True 15 | supports_delete = True 16 | 17 | @classmethod 18 | def get_queryset(cls, request): 19 | return Stage.objects 20 | 21 | StageView.extra_columns = [] -------------------------------------------------------------------------------- /vespene/views/user.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018, Michael DeHaan LLC 2 | # License: Apache License Version 2.0 3 | 4 | from vespene.views import forms 5 | from django.contrib.auth.models import User 6 | from vespene.views import BaseView 7 | from django.conf import settings 8 | from django.core.exceptions import PermissionDenied 9 | 10 | class UserView(BaseView): 11 | model = User 12 | form = forms.UserForm 13 | view_prefix = 'user' 14 | object_label = 'User' 15 | supports_new = True 16 | supports_edit = True 17 | supports_delete = True 18 | 19 | @classmethod 20 | def name_cell(cls, obj): 21 | return obj.username 22 | 23 | @classmethod 24 | def ordering(cls, queryset): 25 | """ 26 | Returns the queryset in the default sort order 27 | """ 28 | return queryset.order_by('username') 29 | 30 | @classmethod 31 | def get_queryset(cls, request): 32 | if settings.ALLOW_UI_USER_CREATION: 33 | return User.objects 34 | else: 35 | raise PermissionDenied 36 | 37 | #@classmethod 38 | #def description_column(cls, obj): 39 | # return obj.description 40 | 41 | UserView.extra_columns = [ 42 | #('Description', SnippetView.description_column) 43 | ] 44 | -------------------------------------------------------------------------------- /vespene/views/variable_set.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018, Michael DeHaan LLC 2 | # License: Apache License Version 2.0 3 | 4 | 5 | from vespene.views import forms 6 | from vespene.models.variable_set import VariableSet 7 | from vespene.views import BaseView 8 | import html 9 | 10 | class VariableSetView(BaseView): 11 | model = VariableSet 12 | form = forms.VariableSetForm 13 | view_prefix = 'variable_set' 14 | object_label = 'Variable Set' 15 | supports_new = True 16 | supports_edit = True 17 | supports_delete = True 18 | 19 | @classmethod 20 | def get_queryset(cls, request): 21 | return VariableSet.objects 22 | 23 | @classmethod 24 | def description_column(cls, obj): 25 | return html.escape(obj.description) 26 | 27 | VariableSetView.extra_columns = [ 28 | ('Description', VariableSetView.description_column) 29 | ] 30 | -------------------------------------------------------------------------------- /vespene/views/worker_pool.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018, Michael DeHaan LLC 2 | # License: Apache License Version 2.0 3 | 4 | from vespene.views import forms 5 | from vespene.models.worker_pool import WorkerPool 6 | from vespene.views import BaseView 7 | 8 | class WorkerPoolView(BaseView): 9 | model = WorkerPool 10 | form = forms.WorkerPoolForm 11 | view_prefix = 'worker_pool' 12 | object_label = 'Worker Pool' 13 | supports_new = True 14 | supports_edit = True 15 | supports_delete = True 16 | 17 | @classmethod 18 | def get_queryset(cls, request): 19 | return WorkerPool.objects 20 | 21 | WorkerPoolView.extra_columns = [] -------------------------------------------------------------------------------- /vespene/workers/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018, Michael DeHaan LLC 2 | # License: Apache License Version 2.0 3 | -------------------------------------------------------------------------------- /vespene/workers/commands.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018, Michael DeHaan LLC 2 | # License: Apache License Version 2.0 3 | # ------------------------------------------------------------------------- 4 | # commands.py - wrappers around executing shell commands 5 | # -------------------------------------------------------------------------- 6 | 7 | import io 8 | import subprocess 9 | import tempfile 10 | import re 11 | import json 12 | import shlex 13 | import shutil 14 | import os 15 | 16 | TIMEOUT = -1 # name of timeout command 17 | 18 | ansi_escape = re.compile(r'(\x9B|\x1B\[)[0-?]*[ -/]*[@-~]') 19 | 20 | import jinja2 21 | 22 | from vespene.common.logger import Logger 23 | from vespene.common.templates import Environment 24 | from vespene.models.build import ABORTED, ABORTING, FAILURE, Build 25 | 26 | LOG = Logger() 27 | 28 | def check_if_can_continue(build): 29 | polled = Build.objects.get(pk=build.id) 30 | if polled.status == ABORTING: 31 | build.status = ABORTED 32 | build.save(force_update=True) 33 | raise Exception("Aborted") 34 | 35 | def handle_output_variables(build, line): 36 | incoming = build.output_variables 37 | if not incoming: 38 | variables = dict() 39 | else: 40 | variables = json.loads(incoming) 41 | tokens = shlex.split(line) 42 | if len(tokens) != 3: 43 | return 44 | k = tokens[1] 45 | v = tokens[2] 46 | variables[k] = v 47 | build.output_variables = json.dumps(variables) 48 | build.save() 49 | 50 | def get_timeout(): 51 | 52 | global TIMEOUT 53 | if TIMEOUT != -1: 54 | return TIMEOUT 55 | if shutil.which("timeout"): 56 | # normal coreutils 57 | TIMEOUT = "timeout" 58 | elif shutil.which("gtimeout"): 59 | # homebrew coreutils 60 | TIMEOUT = "gtimeout" 61 | else: 62 | TIMEOUT = None 63 | return TIMEOUT 64 | 65 | def execute_command(build, command, input_text=None, env=None, log_command=True, output_log=True, message_log=False, timeout=None): 66 | """ 67 | Execute a command (a list or string) with input_text as input, appending 68 | the output of all commands to the build log. 69 | """ 70 | 71 | timeout_cmd = get_timeout() 72 | 73 | shell = True 74 | if type(command) == list: 75 | if timeout and timeout_cmd: 76 | command.insert(0, timeout) 77 | command.insert(0, timeout_cmd) 78 | shell = False 79 | else: 80 | if timeout and timeout_cmd: 81 | command = "%s %s %s" % (timeout_cmd, timeout, command) 82 | 83 | sock = os.environ.get('SSH_AUTH_SOCK', None) 84 | if env and sock: 85 | env['SSH_AUTH_SOCK'] = sock 86 | 87 | if log_command: 88 | LOG.debug("executing: %s" % command) 89 | if build: 90 | build.append_message(command) 91 | 92 | process = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=shell, env=env) 93 | 94 | if input_text is None: 95 | input_text = "" 96 | 97 | stdin = io.TextIOWrapper( 98 | process.stdin, 99 | encoding='utf-8', 100 | line_buffering=True, 101 | ) 102 | stdout = io.TextIOWrapper( 103 | process.stdout, 104 | encoding='utf-8', 105 | ) 106 | stdin.write(input_text) 107 | stdin.close() 108 | 109 | out = "" 110 | for line in stdout: 111 | 112 | line = ansi_escape.sub('', line) 113 | 114 | if build: 115 | check_if_can_continue(build) 116 | if output_log: 117 | build.append_output(line) 118 | if message_log: 119 | build.append_message(line) 120 | out = "" + line 121 | 122 | if line.startswith("vespene/set"): 123 | handle_output_variables(build, line) 124 | 125 | process.wait() 126 | 127 | if process.returncode != 0: 128 | build.append_message("build failed with exit code %s" % process.returncode) 129 | build.status = FAILURE 130 | build.return_code = process.returncode 131 | build.save(force_update=True) 132 | raise Exception("Failed") 133 | return out 134 | 135 | def answer_file(answer): 136 | (fd, fname) = tempfile.mkstemp() 137 | fh = open(fname, "w") 138 | fh.write("#!/bin/bash\n") 139 | fh.write("echo %s" % answer) 140 | fh.close() 141 | os.close(fd) 142 | os.chmod(fname, 0o700) 143 | return fname 144 | -------------------------------------------------------------------------------- /vespene/workers/isolation.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018, Michael DeHaan LLC 2 | # License: Apache License Version 2.0 3 | # ------------------------------------------------------------------------- 4 | # isolation.py - when each build runs, it can be wrapped 5 | # in an isolation procedure to prevent the build from doing "too many 6 | # dangeorus things". This logic is maintained in 'plugins/isolation' and 7 | # can include a simple sudo before execution or building inside a container. 8 | # -------------------------------------------------------------------------- 9 | 10 | from vespene.common.logger import Logger 11 | from vespene.common.plugin_loader import PluginLoader 12 | 13 | LOG = Logger() 14 | 15 | # ============================================================================= 16 | 17 | class IsolationManager(object): 18 | 19 | def __init__(self, builder, build): 20 | self.builder = builder 21 | self.build = build 22 | self.project = self.build.project 23 | self.working_dir = self.build.working_dir 24 | self.isolation = self.project.worker_pool.isolation_method 25 | self.plugin_loader = PluginLoader() 26 | self.provider = self.get_provider() 27 | 28 | # ------------------------------------------------------------------------- 29 | 30 | def get_provider(self): 31 | """ 32 | Return the management object for the given repo type. 33 | """ 34 | plugins = self.plugin_loader.get_isolation_plugins() 35 | plugin = plugins.get(self.isolation) 36 | if plugin is None: 37 | raise Exception("no isolation plugin configurated for worker pool isolation type: %s" % self.isolation) 38 | plugin.setup(self.build) 39 | return plugin 40 | 41 | # ------------------------------------------------------------------------- 42 | 43 | def begin(self): 44 | """ 45 | Begin isolation (chroot, container, sudo, etc) 46 | """ 47 | self.provider.begin() 48 | 49 | # ------------------------------------------------------------------------- 50 | 51 | def execute(self): 52 | """ 53 | Code that launches the build 54 | """ 55 | return self.provider.execute() 56 | 57 | 58 | # ------------------------------------------------------------------------- 59 | 60 | def end(self): 61 | """ 62 | End isolation 63 | """ 64 | return self.provider.end() 65 | -------------------------------------------------------------------------------- /vespene/workers/registration.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018, Michael DeHaan LLC 2 | # License: Apache License Version 2.0 3 | # ------------------------------------------------------------------------- 4 | # registration.py - updates the database to say who is building something 5 | # and what the current settings are, which is used by the file serving 6 | # code to see if it is ok to serve up files in the buildroot. But also 7 | # for record keeping. 8 | # -------------------------------------------------------------------------- 9 | 10 | 11 | from datetime import datetime 12 | import random 13 | import fcntl 14 | import subprocess 15 | import os 16 | 17 | from django.utils import timezone 18 | from django.conf import settings 19 | 20 | from vespene.common.logger import Logger 21 | from vespene.models.worker import Worker 22 | 23 | LOG = Logger() 24 | 25 | WORKER_ID_FILE = "/etc/vespene/worker_id" 26 | 27 | # ============================================================================= 28 | 29 | class RegistrationManager(object): 30 | 31 | def __init__(self, builder, build): 32 | self.builder = builder 33 | self.build = build 34 | self.project = self.build.project 35 | 36 | def create_worker_id(self): 37 | wid = ''.join(random.SystemRandom().choice('abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*(-_=+)') for i in range(50)) 38 | fd = open(WORKER_ID_FILE, "w+") 39 | fd.write(wid) 40 | fd.close() 41 | return wid 42 | 43 | def get_worker_id(self, fd): 44 | return fd.readlines()[0].strip() 45 | 46 | def get_worker_record(self, worker_id): 47 | qs = Worker.objects.filter(worker_uid=worker_id) 48 | if not qs.exists(): 49 | return None 50 | return qs.first() 51 | 52 | # worker_pool = models.ForeignKey('WorkerPool', null=False, on_delete=models.SET_NULL) 53 | # hostname = models.CharField(max_length=1024, null=True) 54 | # port = models.IntField(null=False, default=8080) 55 | # working_dir = models.CharField(max_length=1024, null=True) 56 | # first_checkin = models.DateTimeField(null=True, blank=True) 57 | # last_checkin = models.DateTimeField(null=True, blank=True) 58 | # fileserving_enabled = models.BooleanField(null=False, default=False) 59 | 60 | def get_hostname(self): 61 | if settings.FILESERVING_HOSTNAME: 62 | return settings.FILESERVING_HOSTNAME 63 | return self.guess_hostname() 64 | 65 | def guess_hostname(self): 66 | return subprocess.check_output("hostname").decode('utf-8').strip() 67 | 68 | def get_port(self): 69 | if settings.FILESERVING_PORT: 70 | return settings.FILESERVING_PORT 71 | else: 72 | return 8000 73 | 74 | def get_build_root(self): 75 | return settings.BUILD_ROOT 76 | 77 | def get_fileserving_enabled(self): 78 | return settings.FILESERVING_ENABLED 79 | 80 | def create_worker_record(self, worker_id): 81 | now = datetime.now(tz=timezone.utc) 82 | obj = Worker( 83 | worker_uid = worker_id, 84 | hostname = self.get_hostname(), 85 | port = self.get_port(), 86 | build_root = self.get_build_root(), 87 | first_checkin = now, 88 | last_checkin = now, 89 | fileserving_enabled = self.get_fileserving_enabled() 90 | ) 91 | obj.save() 92 | return obj 93 | 94 | def update_worker_record(self, worker): 95 | now = datetime.now(tz=timezone.utc) 96 | worker.hostname = self.get_hostname() 97 | worker.port = self.get_port() 98 | worker.build_root = self.get_build_root() 99 | worker.last_checkin = now 100 | worker.fileserving_enabled = self.get_fileserving_enabled() 101 | worker.save() 102 | return worker 103 | 104 | def go(self): 105 | """ 106 | Trigger next stage of pipeline if build was successful 107 | """ 108 | 109 | if not os.path.exists(WORKER_ID_FILE): 110 | worker_id = self.create_worker_id() 111 | 112 | fd = open(WORKER_ID_FILE, "r") 113 | fcntl.flock(fd, fcntl.LOCK_EX) 114 | worker_id = self.get_worker_id(fd) 115 | 116 | worker_record = self.get_worker_record(worker_id) 117 | if not worker_record: 118 | worker_record = self.create_worker_record(worker_id) 119 | else: 120 | worker_record = self.update_worker_record(worker_record) 121 | 122 | self.build.worker = worker_record 123 | self.build.save() 124 | fcntl.flock(fd, fcntl.LOCK_UN) 125 | -------------------------------------------------------------------------------- /vespene/workers/scm.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018, Michael DeHaan LLC 2 | # License: Apache License Version 2.0 3 | # ------------------------------------------------------------------------- 4 | # scm.py - encapsulates the logic of a SCM checkout. The implementation 5 | # for various source control providers is contained in 'plugins/scm'. 6 | # -------------------------------------------------------------------------- 7 | 8 | from vespene.common.logger import Logger 9 | from vespene.common.plugin_loader import PluginLoader 10 | from vespene.plugins.scm.none import Plugin as NoScmPlugin 11 | LOG = Logger() 12 | 13 | # ============================================================================= 14 | 15 | class ScmManager(object): 16 | 17 | def __init__(self, builder, build): 18 | self.builder = builder 19 | self.build = build 20 | self.project = self.build.project 21 | self.repo = self.project.repo_url 22 | self.scm_type = self.project.scm_type 23 | 24 | self.plugin_loader = PluginLoader() 25 | self.provider = self.get_provider() 26 | 27 | # ------------------------------------------------------------------------- 28 | 29 | def get_provider(self): 30 | """ 31 | Return the management object for the given repo type. 32 | """ 33 | plugins = self.plugin_loader.get_scm_plugins() 34 | plugin = plugins.get(self.scm_type) 35 | if plugin is None: 36 | plugin = NoScmPlugin() 37 | plugin.setup(self.build) 38 | return plugin 39 | 40 | # ------------------------------------------------------------------------- 41 | 42 | def checkout(self): 43 | """ 44 | Perform a checkout in the already configured build dir 45 | """ 46 | self.provider.checkout() 47 | 48 | # ------------------------------------------------------------------------- 49 | 50 | def get_revision(self): 51 | """ 52 | Find out what the source control revision is. 53 | """ 54 | return self.provider.get_revision() 55 | 56 | # ------------------------------------------------------------------------- 57 | 58 | def get_last_commit_user(self): 59 | """ 60 | Find out what user made the last commit on this branch 61 | """ 62 | return self.provider.get_last_commit_user() 63 | -------------------------------------------------------------------------------- /vespene/workers/ssh_agent.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018, Michael DeHaan LLC 2 | # License: Apache License Version 2.0 3 | # ------------------------------------------------------------------------- 4 | # ssh_agent.py - Vespene workers are run wrapped by 'ssh-agent' processes 5 | # and the workers can use SSH keys configured per project to do SCM checkouts 6 | # or use SSH-based automation. This is mostly handled right now 7 | # through basic expect scripts and does support password-locked keys. 8 | # -------------------------------------------------------------------------- 9 | 10 | import os 11 | import tempfile 12 | 13 | from vespene.common.logger import Logger 14 | from vespene.workers import commands 15 | 16 | LOG = Logger() 17 | 18 | # ============================================================================= 19 | 20 | class SshAgentManager(object): 21 | 22 | def __init__(self, builder, build): 23 | self.builder = builder 24 | self.build = build 25 | self.project = self.build.project 26 | self.tempfile_paths = [] 27 | 28 | def add_all_keys(self): 29 | for access in self.project.ssh_keys.all(): 30 | self.add_key(access) 31 | 32 | def add_key(self, access): 33 | 34 | (_, keyfile) = tempfile.mkstemp() 35 | answer_file = None 36 | 37 | try: 38 | fh = open(keyfile, "w") 39 | private = access.get_private_key() 40 | fh.write(private) 41 | fh.close() 42 | 43 | answer_file = None 44 | 45 | 46 | if access.unlock_password: 47 | LOG.debug("adding SSH key with passphrase!") 48 | self.ssh_add_with_passphrase(keyfile, access.get_unlock_password()) 49 | else: 50 | if ',ENCRYPTED' in private: 51 | raise Exception("SSH key has a passphrase but an unlock password was not set. Aborting") 52 | LOG.debug("adding SSH key without passphrase!") 53 | self.ssh_add_without_passphrase(keyfile) 54 | finally: 55 | os.remove(keyfile) 56 | if answer_file: 57 | os.remove(answer_file) 58 | 59 | def cleanup(self): 60 | # remove SSH identities 61 | LOG.debug("removing SSH identities") 62 | commands.execute_command(self.build, "ssh-add -D", log_command=False, message_log=False, output_log=False) 63 | 64 | def ssh_add_without_passphrase(self, keyfile): 65 | LOG.debug(keyfile) 66 | cmd = "ssh-add %s < /dev/null" % keyfile 67 | commands.execute_command(self.build, cmd, env=None, log_command=False, message_log=False, output_log=False) 68 | 69 | def ssh_add_with_passphrase(self, keyfile, passphrase): 70 | (_, fname) = tempfile.mkstemp() 71 | fh = open(fname, "w") 72 | script = """ 73 | #!/usr/bin/expect -f 74 | spawn ssh-add %s 75 | expect "Enter passphrase*:" 76 | send "%s\n"; 77 | expect "Identity added*" 78 | interact 79 | """ % (keyfile, passphrase) 80 | fh.write(script) 81 | fh.close() 82 | commands.execute_command(self.build, "/usr/bin/expect -f %s" % fname, output_log=False, message_log=False) 83 | os.remove(fname) 84 | return fname 85 | -------------------------------------------------------------------------------- /vespene/workers/triggers.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018, Michael DeHaan LLC 2 | # License: Apache License Version 2.0 3 | # ------------------------------------------------------------------------- 4 | # triggers.py - triggers are plugins that are run when certain things happen, 5 | # like when a build starts, or stops successfully or unsuccessfully. Triggers 6 | # are plugins - the basic command trigger can execute arbitrary CLI commands. 7 | # -------------------------------------------------------------------------- 8 | 9 | from vespene.models.build import SUCCESS 10 | from vespene.common.plugin_loader import PluginLoader 11 | 12 | # ========================================================= 13 | 14 | class TriggerManager(object): 15 | 16 | """ 17 | Trigger manager handles activation of pre and post build 18 | triggers. 19 | """ 20 | 21 | # ----------------------------------------------------- 22 | 23 | def __init__(self, builder, build): 24 | """ 25 | Constructor takes a build reference. 26 | """ 27 | self.builder = builder 28 | self.build = build 29 | 30 | self.plugin_loader = PluginLoader() 31 | self.pre_trigger_plugins = self.plugin_loader.get_pre_trigger_plugins() 32 | self.success_trigger_plugins = self.plugin_loader.get_success_trigger_plugins() 33 | self.failure_trigger_plugins = self.plugin_loader.get_failure_trigger_plugins() 34 | 35 | # ----------------------------------------------------- 36 | 37 | def run_all_pre(self): 38 | """ 39 | Run all pre hooks - which can be scripts 40 | that simply take command line flags or recieve 41 | more detail on standard input. See docs for details. 42 | """ 43 | self.build.append_message("----------\nPre hooks...") 44 | context = self.pre_context() 45 | for plugin in self.pre_trigger_plugins: 46 | plugin.execute_hook(self.build, context) 47 | 48 | # ----------------------------------------------------- 49 | 50 | def run_all_post(self): 51 | """ 52 | Similar to post hooks, pre hooks can be set to run 53 | only on success or failure. 54 | """ 55 | self.build.append_message("----------\nPost hooks...") 56 | context = self.post_context() 57 | 58 | if self.build.status == SUCCESS: 59 | for plugin in self.success_trigger_plugins: 60 | plugin.execute_hook(self.build, context) 61 | 62 | else: 63 | for plugin in self.failure_trigger_plugins: 64 | plugin.execute_hook(self.build, context) 65 | 66 | # ----------------------------------------------------- 67 | 68 | def pre_context(self): 69 | """ 70 | This dictionary is passed as JSON on standard 71 | input to pre hooks. 72 | """ 73 | return dict( 74 | hook='pre', 75 | build=self.build, 76 | project=self.build.project 77 | ) 78 | 79 | # ----------------------------------------------------- 80 | 81 | def post_context(self): 82 | """ 83 | This dictionary is passed as JSON on standard 84 | input to post hooks. 85 | """ 86 | return dict( 87 | hook='post', 88 | build=self.build, 89 | project=self.build.project 90 | ) 91 | -------------------------------------------------------------------------------- /vespene/wsgi.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018, Michael DeHaan LLC 2 | # License: Apache License Version 2.0 3 | # ------------------------------------------------------------------------- 4 | # wsgi.py - standard wsgi entry point generated from Django. Not used if 5 | # running the default webserver. 6 | # -------------------------------------------------------------------------- 7 | 8 | import os 9 | 10 | from django.core.wsgi import get_wsgi_application 11 | 12 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "vespene.settings") 13 | 14 | application = get_wsgi_application() 15 | --------------------------------------------------------------------------------