├── bsdploy ├── tests │ ├── __init__.py │ ├── packagesite.txz │ ├── test_templates.py │ ├── test_bootstrap_daemonology.py │ ├── conftest.py │ ├── test_roles.py │ ├── test_quickstart.py │ ├── test_bootstrap_mfsbsd.py │ └── test_bsdploy.py ├── bootstrap-files │ ├── make.conf │ ├── pf.conf │ ├── sshd_config │ ├── FreeBSD.conf │ ├── pkg.conf │ ├── rc.conf │ ├── daemonology-files.yml │ └── files.yml ├── roles │ ├── jails_host │ │ ├── files │ │ │ ├── base_flavour_motd │ │ │ ├── make.conf │ │ │ ├── base_flavour_sshd_config │ │ │ └── base_flavour_rc.conf │ │ ├── templates │ │ │ ├── FreeBSD.conf │ │ │ ├── pkg.conf │ │ │ ├── pf.conf │ │ │ └── ezjail.conf │ │ ├── handlers │ │ │ └── main.yml │ │ ├── tasks │ │ │ ├── ntpd.yml │ │ │ ├── pf.yml │ │ │ ├── obsolete-ipf.yml │ │ │ └── main.yml │ │ └── defaults │ │ │ └── main.yml │ ├── dhcp_host │ │ ├── defaults │ │ │ └── main.yml │ │ ├── handlers │ │ │ └── main.yml │ │ ├── tasks │ │ │ └── main.yml │ │ └── templates │ │ │ └── dhclient-exit-hooks │ ├── jailhost.yml │ └── zfs_auto_snapshot │ │ ├── templates │ │ └── 010.zfs-snapshot │ │ └── tasks │ │ └── main.yml ├── startup-ansible-jail.sh ├── enable_root_login_on_daemonology.sh ├── download.py ├── fabfile_daemonology.py ├── library │ ├── sysrc │ └── zpool ├── fabfile_digitalocean.py ├── fabutils.py ├── __init__.py └── fabfile_mfsbsd.py ├── docs ├── .gitignore ├── tutorial │ ├── overview.rst │ ├── staging.rst │ ├── transmission.rst │ └── webserver.rst ├── advanced │ ├── updating.rst │ ├── customizing-bootstrap.rst │ └── staging.rst ├── index.rst ├── usage │ ├── jails.rst │ ├── ansible-with-fabric.rst │ ├── fabric.rst │ └── ansible.rst ├── setup │ ├── overview.rst │ ├── provisioning-virtualbox.rst │ ├── configuration.rst │ ├── provisioning-plain.rst │ ├── provisioning-ec2.rst │ └── bootstrapping.rst ├── installation.rst ├── Makefile └── conf.py ├── .coveragerc ├── setup.cfg ├── .gitignore ├── default.nix ├── MANIFEST.in ├── Makefile ├── tox.ini ├── LICENSE.txt ├── setup.py ├── .github └── workflows │ └── main.yml ├── README.rst └── CHANGES.rst /bsdploy/tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docs/.gitignore: -------------------------------------------------------------------------------- 1 | /_build/ 2 | -------------------------------------------------------------------------------- /bsdploy/bootstrap-files/make.conf: -------------------------------------------------------------------------------- 1 | WITH_PKGNG=yes 2 | -------------------------------------------------------------------------------- /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | branch = True 3 | include = bsdploy/* 4 | -------------------------------------------------------------------------------- /bsdploy/roles/jails_host/files/base_flavour_motd: -------------------------------------------------------------------------------- 1 | Went to Jail. -------------------------------------------------------------------------------- /bsdploy/roles/jails_host/files/make.conf: -------------------------------------------------------------------------------- 1 | WITH_PKGNG=yes 2 | -------------------------------------------------------------------------------- /bsdploy/bootstrap-files/pf.conf: -------------------------------------------------------------------------------- 1 | pass in all 2 | pass out all 3 | -------------------------------------------------------------------------------- /bsdploy/roles/dhcp_host/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | dhcp_host_sshd_interface: "" 3 | -------------------------------------------------------------------------------- /bsdploy/roles/jailhost.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - user: root 3 | roles: 4 | - dhcp_host 5 | - jails_host 6 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [bdist_wheel] 2 | universal = 1 3 | 4 | [devpi:upload] 5 | formats = sdist.tgz,bdist_wheel 6 | -------------------------------------------------------------------------------- /bsdploy/tests/packagesite.txz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ployground/bsdploy/HEAD/bsdploy/tests/packagesite.txz -------------------------------------------------------------------------------- /bsdploy/roles/jails_host/templates/FreeBSD.conf: -------------------------------------------------------------------------------- 1 | FreeBSD: { 2 | enabled: no 3 | } 4 | {{ ploy_jail_host_pkg_repository }} 5 | -------------------------------------------------------------------------------- /bsdploy/bootstrap-files/sshd_config: -------------------------------------------------------------------------------- 1 | Port 22 2 | PermitRootLogin without-password 3 | Subsystem sftp /usr/libexec/sftp-server 4 | UseDNS no 5 | -------------------------------------------------------------------------------- /bsdploy/roles/dhcp_host/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: renew dhcp lease 3 | raw: /sbin/dhclient {{ ansible_default_ipv4.interface }} || true 4 | -------------------------------------------------------------------------------- /bsdploy/roles/jails_host/files/base_flavour_sshd_config: -------------------------------------------------------------------------------- 1 | PermitRootLogin without-password 2 | Subsystem sftp /usr/libexec/sftp-server 3 | UseDNS no 4 | -------------------------------------------------------------------------------- /bsdploy/roles/jails_host/files/base_flavour_rc.conf: -------------------------------------------------------------------------------- 1 | sshd_enable="YES" 2 | rpcbind_enable="NO" 3 | cron_flags="$cron_flags -J 15" 4 | syslogd_flags="-ss" 5 | -------------------------------------------------------------------------------- /bsdploy/roles/zfs_auto_snapshot/templates/010.zfs-snapshot: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | PATH=$PATH:/sbin:/usr/local/bin /usr/local/sbin/zfs-auto-snapshot -u {{ item.name }} {{ item.num_keep }} 3 | -------------------------------------------------------------------------------- /bsdploy/startup-ansible-jail.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | exec 1>/var/log/startup.log 2>&1 3 | chmod 0600 /var/log/startup.log 4 | set -e 5 | set -x 6 | pkg update 7 | pkg upgrade 8 | pkg install python2 9 | -------------------------------------------------------------------------------- /bsdploy/bootstrap-files/FreeBSD.conf: -------------------------------------------------------------------------------- 1 | FreeBSD: { 2 | enabled: no 3 | } 4 | FreeBSD-custom: { 5 | url: "{{ploy_jail_host_pkg_repository}}", 6 | mirror_type: "srv", 7 | enabled: yes 8 | } 9 | -------------------------------------------------------------------------------- /bsdploy/roles/dhcp_host/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Create dhclient-exit-hooks 3 | template: src=dhclient-exit-hooks dest=/etc/dhclient-exit-hooks 4 | notify: renew dhcp lease 5 | tags: dhcp_host 6 | -------------------------------------------------------------------------------- /bsdploy/bootstrap-files/pkg.conf: -------------------------------------------------------------------------------- 1 | ASSUME_ALWAYS_YES: YES 2 | REPO_AUTOUPDATE: NO 3 | {% if http_proxy is defined %} 4 | pkg_env: { 5 | HTTP_PROXY: "{{http_proxy}}", 6 | HTTPS_PROXY: "{{http_proxy}}" 7 | } 8 | {% endif %} 9 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /bin/ 2 | /build/ 3 | /develop-eggs/ 4 | /dist/ 5 | /eggs/ 6 | /include/ 7 | /htmlcov/ 8 | /lib/ 9 | /share/ 10 | /src/ 11 | /.cache 12 | /.coverage 13 | /.installed.cfg 14 | /.mr.developer.cfg 15 | /.tox 16 | 17 | -------------------------------------------------------------------------------- /bsdploy/bootstrap-files/rc.conf: -------------------------------------------------------------------------------- 1 | hostname="{{hostname}}" 2 | sshd_enable="YES" 3 | syslogd_flags="-ss" 4 | zfs_enable="YES" 5 | pf_enable="YES" 6 | {% for interface in interfaces %} 7 | ifconfig_{{interface}}="DHCP" 8 | {% endfor %} 9 | -------------------------------------------------------------------------------- /bsdploy/roles/jails_host/templates/pkg.conf: -------------------------------------------------------------------------------- 1 | ASSUME_ALWAYS_YES: YES 2 | REPO_AUTOUPDATE: NO 3 | {% if ploy_http_proxy is defined %} 4 | pkg_env: { 5 | HTTP_PROXY: "{{ploy_http_proxy}}", 6 | HTTPS_PROXY: "{{ploy_http_proxy}}" 7 | } 8 | {% endif %} 9 | -------------------------------------------------------------------------------- /bsdploy/bootstrap-files/daemonology-files.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 'make.conf': 3 | remote: '/etc/make.conf' 4 | 'pkg.conf': 5 | remote: '/usr/local/etc/pkg.conf' 6 | 'FreeBSD.conf': 7 | directory: '/usr/local/etc/pkg/repos' 8 | remote: '/usr/local/etc/pkg/repos/FreeBSD.conf' 9 | -------------------------------------------------------------------------------- /default.nix: -------------------------------------------------------------------------------- 1 | with import {}; 2 | stdenv.mkDerivation rec { 3 | name = "env"; 4 | env = buildEnv { name = name; paths = buildInputs; }; 5 | buildInputs = [ 6 | python 7 | python27Packages.pynacl 8 | gnumake 9 | openssl 10 | ]; 11 | } 12 | -------------------------------------------------------------------------------- /bsdploy/roles/jails_host/templates/pf.conf: -------------------------------------------------------------------------------- 1 | {% for network in pf_nat_jail_networks %} 2 | nat on {{ pf_nat_interface }} from {{ network }} to any -> ({{ pf_nat_interface }}:0) 3 | {% endfor %} 4 | {% for rule in pf_nat_rules %} 5 | {{ rule }} 6 | {% endfor %} 7 | pass in all 8 | pass out all 9 | -------------------------------------------------------------------------------- /bsdploy/enable_root_login_on_daemonology.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | mkdir /root/.ssh 3 | chmod 700 /root/.ssh 4 | chmod 600 /tmp/authorized_keys 5 | mv /tmp/authorized_keys /root/.ssh/ 6 | chown root /root/.ssh/authorized_keys 7 | echo "PermitRootLogin without-password" >> /etc/ssh/sshd_config 8 | /etc/rc.d/sshd fastreload -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.rst 2 | include CHANGES.rst 3 | include setup.py 4 | include tox.ini 5 | include bsdploy/*.sh 6 | include bsdploy/tests/*.py 7 | include bsdploy/tests/*.txz 8 | recursive-include bsdploy/bootstrap-files * 9 | recursive-include bsdploy/library * 10 | recursive-include bsdploy/roles * 11 | -------------------------------------------------------------------------------- /bsdploy/tests/test_templates.py: -------------------------------------------------------------------------------- 1 | def test_dhcp_host_dhclient_exit_hooks(): 2 | pass 3 | 4 | 5 | def test_jails_host_pf_conf(): 6 | pass 7 | 8 | 9 | def test_jails_host_ezjail_conf(): 10 | pass 11 | 12 | 13 | def test_jails_host_freebsd_conf(): 14 | pass 15 | 16 | 17 | def test_zfs_auto_snapshot_010_zfs_snapshot(): 18 | pass 19 | 20 | 21 | def test_jails_host_pkg_conf(): 22 | pass 23 | -------------------------------------------------------------------------------- /bsdploy/roles/jails_host/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: restart network 3 | raw: /etc/netstart 4 | - name: restart ezjail 5 | service: name=ezjail state=restarted 6 | - name: reload pf 7 | service: name=pf state=reloaded 8 | - name: restart sshd 9 | service: name=sshd state=restarted 10 | - name: restart ntpd 11 | service: name=ntpd state=restarted 12 | - name: reload sysctl 13 | raw: service sysctl reload 14 | register: command_result 15 | failed_when: "command_result.stderr" 16 | -------------------------------------------------------------------------------- /bsdploy/bootstrap-files/files.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 'rc.conf': 3 | remote: '/mnt/etc/rc.conf' 4 | use_jinja: True 5 | 'make.conf': 6 | remote: '/mnt/etc/make.conf' 7 | 'pkg.conf': 8 | remote: '/mnt/usr/local/etc/pkg.conf' 9 | use_jinja: True 10 | 'pf.conf': 11 | remote: '/mnt/etc/pf.conf' 12 | 'FreeBSD.conf': 13 | directory: '/mnt/usr/local/etc/pkg/repos' 14 | remote: '/mnt/usr/local/etc/pkg/repos/FreeBSD.conf' 15 | use_jinja: True 16 | 'sshd_config': 17 | remote: '/mnt/etc/ssh/sshd_config' 18 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # convenience Makefile to setup a development version 2 | # of bsdploy and its direct dependencies 3 | 4 | develop: .installed.cfg 5 | 6 | .installed.cfg: bin/buildout buildout.cfg bin/virtualenv 7 | bin/buildout -v 8 | 9 | bin/buildout: bin/pip 10 | bin/pip install -U "zc.buildout>=3dev" 11 | 12 | # needed for tox 13 | bin/virtualenv: bin/pip 14 | bin/pip install virtualenv 15 | 16 | bin/pip: 17 | virtualenv -p python2.7 . 18 | 19 | clean: 20 | git clean -dxxf 21 | 22 | docs: 23 | make -C docs html 24 | 25 | .PHONY: clean docs 26 | -------------------------------------------------------------------------------- /bsdploy/roles/dhcp_host/templates/dhclient-exit-hooks: -------------------------------------------------------------------------------- 1 | # reset the sshd_config in any case 2 | case "$reason" in 3 | BOUND|RENEW|REBIND|REBOOT) 4 | # if we get a new ip, we add it to sshd_config and reload 5 | [ "$interface" != "" -a "$interface" != "{{ dhcp_host_sshd_interface }}" ] && exit 0 6 | grep -v "^ListenAddress" /etc/ssh/sshd_config > /etc/ssh/sshd_config.dhclientbase 7 | cp /etc/ssh/sshd_config.dhclientbase /etc/ssh/sshd_config 8 | echo "ListenAddress $new_ip_address" >> /etc/ssh/sshd_config 9 | service sshd reload 10 | ;; 11 | esac 12 | -------------------------------------------------------------------------------- /bsdploy/roles/jails_host/tasks/ntpd.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Enable ntpd in rc.conf 3 | service: 4 | name: ntpd 5 | enabled: yes 6 | notify: restart ntpd 7 | tags: ntpd 8 | 9 | - name: Disable public use of ntpd 10 | copy: 11 | content: | 12 | server 0.freebsd.pool.ntp.org iburst 13 | server 1.freebsd.pool.ntp.org iburst 14 | server 2.freebsd.pool.ntp.org iburst 15 | restrict default kod nomodify notrap nopeer noquery 16 | restrict -6 default kod nomodify notrap nopeer noquery 17 | restrict 127.0.0.1 18 | restrict -6 ::1 19 | restrict 127.127.1.0 20 | dest: /etc/ntp.conf 21 | notify: restart ntpd 22 | tags: ntpd 23 | -------------------------------------------------------------------------------- /bsdploy/download.py: -------------------------------------------------------------------------------- 1 | import hashlib 2 | import os 3 | import sys 4 | import urllib 5 | 6 | 7 | def check(path, sha): 8 | d = hashlib.sha1() 9 | with open(path, 'rb') as f: 10 | while 1: 11 | buf = f.read(1024 * 1024) 12 | if not len(buf): 13 | break 14 | d.update(buf) 15 | return d.hexdigest() == sha 16 | 17 | 18 | def run(*args, **kwargs): 19 | url = sys.argv[1] 20 | sha = sys.argv[2] 21 | path = sys.argv[3] 22 | if os.path.isdir(path): 23 | import urlparse 24 | path = os.path.join(path, urlparse.urlparse(url).path.split('/')[-1]) 25 | if os.path.exists(path): 26 | if not check(path, sha): 27 | os.remove(path) 28 | if not os.path.exists(path): 29 | urllib.urlretrieve(url, path) 30 | if not check(path, sha): 31 | sys.exit(1) 32 | 33 | 34 | if __name__ == '__main__': 35 | run() 36 | -------------------------------------------------------------------------------- /bsdploy/roles/jails_host/tasks/pf.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Enable pf in rc.conf 3 | service: 4 | name: pf 5 | enabled: yes 6 | - name: Check for /etc/pf.conf 7 | stat: 8 | path: /etc/pf.conf 9 | register: stat_etc_pf_conf 10 | - name: Default pf.conf 11 | copy: 12 | content: | 13 | pass in all 14 | pass out all 15 | dest: /etc/pf.conf 16 | when: stat_etc_pf_conf.stat.exists == False 17 | - name: Stat of /dev/pf 18 | stat: 19 | path: /dev/pf 20 | register: stat_dev_pf 21 | - name: Checking pf 22 | fail: 23 | msg: "The pf service is not running, you have to start it manually" 24 | when: stat_dev_pf.stat.exists == False 25 | - name: Setup pf.conf 26 | template: 27 | src: pf.conf 28 | dest: /etc/pf.conf 29 | validate: '/sbin/pfctl -nf %s' 30 | tags: pf-conf 31 | register: setup_pf_conf_result 32 | - name: Reload pf.conf 33 | raw: service pf reload 34 | when: setup_pf_conf_result.changed 35 | tags: pf-conf 36 | 37 | - name: Enable gateway in rc.conf 38 | sysrc: 39 | name: gateway_enable 40 | value: "YES" 41 | notify: restart network 42 | -------------------------------------------------------------------------------- /bsdploy/roles/zfs_auto_snapshot/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Ensure zfstools is installed 3 | pkgng: 4 | name: "zfstools" 5 | state: "present" 6 | tags: zfs_auto_snapshot 7 | 8 | - name: Setup hourly zfs-auto-snapshot 9 | cron: name="hourly zfs-auto-snapshot" special_time=hourly job="PATH=$PATH:/sbin:/usr/local/bin /usr/local/sbin/zfs-auto-snapshot -u hourly 24" 10 | tags: zfs_auto_snapshot 11 | - name: "Setup {{ item.name }} zfs-auto-snapshot" 12 | template: 13 | src: 010.zfs-snapshot 14 | dest: "/etc/periodic/{{ item.name }}/010.zfs-snapshot" 15 | mode: "0755" 16 | with_items: 17 | - { name: daily, num_keep: 7 } 18 | - { name: weekly, num_keep: 4 } 19 | - { name: monthly, num_keep: 12 } 20 | tags: zfs_auto_snapshot 21 | 22 | - name: Cleanup old daily cron zfs snapshot 23 | cron: name="daily zfs-auto-snapshot" state=absent 24 | tags: zfs_auto_snapshot 25 | - name: Cleanup old weekly cron zfs snapshot 26 | cron: name="weekly zfs-auto-snapshot" state=absent 27 | tags: zfs_auto_snapshot 28 | - name: Cleanup old monthly cron zfs snapshot 29 | cron: name="monthly zfs-auto-snapshot" state=absent 30 | tags: zfs_auto_snapshot 31 | -------------------------------------------------------------------------------- /bsdploy/roles/jails_host/tasks/obsolete-ipf.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Check for old ipnat_rules setting 3 | fail: 4 | msg: "ipnat_rules is not supported anymore, see documentation on pf_nat_rules" 5 | when: ipnat_rules is defined 6 | tags: pf-conf 7 | 8 | - name: Remove ipfilter from rc.conf 9 | lineinfile: 10 | dest: /etc/rc.conf 11 | regexp: ^ipfilter_enable= 12 | state: absent 13 | notify: restart network 14 | - name: Remove ipfilter_rules from rc.conf 15 | lineinfile: 16 | dest: /etc/rc.conf 17 | regexp: ^ipfilter_rules= 18 | state: absent 19 | notify: restart network 20 | 21 | - name: Remove ipmon from rc.conf 22 | lineinfile: 23 | dest: /etc/rc.conf 24 | regexp: ^ipmon_enable= 25 | state: absent 26 | notify: restart network 27 | - name: Remove ipmon_flags from rc.conf 28 | lineinfile: 29 | dest: /etc/rc.conf 30 | regexp: ^ipmon_flags= 31 | state: absent 32 | notify: restart network 33 | 34 | - name: Remove ipnat from rc.conf 35 | lineinfile: 36 | dest: /etc/rc.conf 37 | regexp: ^ipnat_enable= 38 | state: absent 39 | notify: restart network 40 | - name: Remove ipnat_rules from rc.conf 41 | lineinfile: 42 | dest: /etc/rc.conf 43 | regexp: ^ipnat_rules= 44 | state: absent 45 | notify: restart network 46 | -------------------------------------------------------------------------------- /docs/tutorial/overview.rst: -------------------------------------------------------------------------------- 1 | Tutorial 2 | ======== 3 | 4 | 5 | Overview 6 | -------- 7 | 8 | This tutorial will focus on setting up a host as an all-purpose webserver, with a dedicated jail for handling the incoming traffic and redirecting it to the appropriate jails. 9 | 10 | In it, we will also demonstrate how to setup a staging/production environment. We will start with developing the host on a virtual instance and at the end will replay the final setup against a 'production' host. 11 | 12 | We will 13 | 14 | - configure a webserver jail (nginx) with a simple static overview site which contains links to the various services offered inside the jails. This step will also demonstrate the integration of Fabric scripts and configuring FreeBSD services and how to mount ZFS filesystems into jails. 15 | - install an application that is more or less usable out-of-the-box and requires minimal configuration (transmission, a bit torrent client with a web frontend) 16 | - create a 'production' configuration and apply the setup to it 17 | 18 | 19 | Requirements 20 | ------------ 21 | 22 | To follow along this tutorial, you will need to have :doc:`installed bsdploy ` and a running jailhost. The easiest and recommended way to achieve this is to follow along the :doc:`quickstart `. 23 | -------------------------------------------------------------------------------- /docs/advanced/updating.rst: -------------------------------------------------------------------------------- 1 | Updating 2 | ======== 3 | 4 | While in theory automated systems such as ploy allow you to create new and up-to-date instances easily and thus in theory you would never have to upgrade existing instances because you would simply replace them with newer versions, in practice both jails and host systems will often have to be upgrade in place. 5 | 6 | To support this, bsdploy provides a few helper tasks which are registered by default for jailhosts. 7 | 8 | If you want to use them in your own, custom fabfile you must import them their to make them available, i.e. like so:: 9 | 10 | from bsdploy.fabutils import * 11 | 12 | You can verify this by running the `do` command with `-l`: 13 | 14 | 15 | # ploy do HOST -l 16 | Available commands: 17 | 18 | pkg_upgrade 19 | rsync 20 | update_flavour_pkg 21 | 22 | You can use the `pkg_upgrade` task to keep the pkg and and the installed packages on the host up-to-date. 23 | 24 | The `update_flavour_pkg` is useful after updating the ezjail world, so that newly created jails will have an updated version of pkg from the start. (if the pkg versions become too far apart it can happen, that new jails won't bootstrap at all, because they already fail at installing python). 25 | 26 | See the `fabutils.py` file for more details. 27 | -------------------------------------------------------------------------------- /bsdploy/roles/jails_host/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | ploy_bootstrap_data_pool_name: "tank" 3 | ploy_bootstrap_raid_mode: "detect" 4 | ploy_bootstrap_data_pool_devices: "" 5 | ploy_bootstrap_geli: no 6 | ploy_bootstrap_zpool_version: 28 7 | ploy_ezjail_install_host: "ftp.freebsd.org" 8 | jails_dir: /usr/jails 9 | ploy_jails_zfs_root: "{{ ploy_bootstrap_data_pool_name }}/jails" 10 | zfs_checksum: fletcher4 11 | pkg_txz_url: http://pkg.freebsd.org/freebsd:10:x86:64/quarterly/Latest/pkg.txz 12 | jails_pkg_url: "pkg+http://pkg.freeBSD.org/${ABI}/latest" 13 | pf_nat_jail_networks: 14 | - 10.0.0.0/8 15 | pf_nat_rules: [] 16 | pf_nat_interface: "{{ansible_default_ipv4.interface}}" 17 | ploy_jail_host_sshd_listenaddress: "{{ ansible_default_ipv4.address }}" 18 | ploy_jail_host_pkg_repository_kind: "quarterly" 19 | ploy_jail_host_pkg_repository_url: "pkg+http://pkg.freeBSD.org/${ABI}/{{ ploy_jail_host_pkg_repository_kind }}" 20 | ploy_jail_host_pkg_repository: | 21 | FreeBSD-custom: { 22 | url: "{{ ploy_jail_host_pkg_repository_url }}", 23 | mirror_type: "srv", 24 | enabled: yes 25 | } 26 | ploy_root_user_name: "{{ploy_user | default('root')}}" 27 | ploy_root_home_path: "{{ '/' if ploy_root_user_name == 'root' else '/usr/home/' }}{{ploy_root_user_name}}" 28 | ploy_jail_host_cloned_interfaces: lo1 29 | ploy_jail_host_default_jail_interface: lo1 30 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = 3 | py27{,-ansible19,-ansible24,-ansible25,-ansible26,-ansible27,-ansible28,-ansible29,-ansible210}, 4 | py37{,-ansible25,-ansible26,-ansible27,-ansible28,-ansible29,-ansible210} 5 | py38{,-ansible25,-ansible26,-ansible27,-ansible28,-ansible29,-ansible210} 6 | py39{,-ansible25,-ansible26,-ansible27,-ansible28,-ansible29,-ansible210} 7 | py310{,-ansible4,-ansible5,-ansible6} 8 | 9 | [testenv] 10 | deps = 11 | ansible19: ansible>=1.9,<2dev 12 | ansible24: ansible>=2.4,<2.5dev 13 | ansible25: ansible>=2.5,<2.6dev 14 | ansible26: ansible>=2.6,<2.7dev 15 | ansible25,ansible26: jinja2<3.1 16 | ansible27: ansible>=2.7,<2.8dev 17 | ansible28: ansible>=2.8,<2.9dev 18 | ansible29: ansible>=2.9,<2.10dev 19 | ansible210: ansible>=2.10,<2.11dev 20 | ansible4: ansible>=4,<5dev 21 | ansible5: ansible>=5,<6dev 22 | ansible6: ansible>=6,<7dev 23 | coverage 24 | flake8<5 25 | mock 26 | ploy_virtualbox>=2dev 27 | pytest 28 | pytest-cov 29 | pytest-flake8 < 1.1.0;python_version=="2.7" 30 | pytest-flake8;python_version!="2.7" 31 | commands = 32 | {envbindir}/py.test --cov {envsitepackagesdir}/bsdploy/ --cov-report html:{toxinidir}/htmlcov_{envname} {posargs} {envsitepackagesdir}/bsdploy/ 33 | 34 | [pytest] 35 | addopts = --flake8 --tb=native -W "ignore:With-statements now directly support multiple context managers:DeprecationWarning" 36 | flake8-ignore = E501 E128 E129 37 | testpaths = bsdploy 38 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2014, Tom Lazar 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 5 | 6 | 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 7 | 8 | 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 9 | 10 | 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 11 | 12 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 13 | -------------------------------------------------------------------------------- /bsdploy/tests/test_bootstrap_daemonology.py: -------------------------------------------------------------------------------- 1 | from bsdploy import bsdploy_path 2 | import pytest 3 | 4 | 5 | @pytest.fixture 6 | def bootstrap(env_mock, environ_mock, monkeypatch, put_mock, run_mock, tempdir, yesno_mock, ployconf): 7 | from bsdploy.fabfile_daemonology import bootstrap 8 | ployconf.fill('') 9 | environ_mock['HOME'] = tempdir.directory 10 | env_mock.host_string = 'jailhost' 11 | monkeypatch.setattr('bsdploy.fabfile_daemonology.env', env_mock) 12 | monkeypatch.setattr('bsdploy.fabfile_daemonology.put', put_mock) 13 | monkeypatch.setattr('bsdploy.fabfile_daemonology.run', run_mock) 14 | monkeypatch.setattr('bsdploy.fabfile_daemonology.sleep', lambda x: None) 15 | return bootstrap 16 | 17 | 18 | def test_bootstrap(bootstrap, capsys, put_mock, run_mock, tempdir, yesno_mock): 19 | format_info = dict( 20 | bsdploy_path=bsdploy_path, 21 | tempdir=tempdir.directory) 22 | put_mock.expected = [ 23 | (('bootstrap-files/authorized_keys', '/tmp/authorized_keys'), {}), 24 | (("%(bsdploy_path)s/enable_root_login_on_daemonology.sh" % format_info, '/tmp/'), {'mode': '0775'}), 25 | (("%(bsdploy_path)s/bootstrap-files/FreeBSD.conf" % format_info, '/usr/local/etc/pkg/repos/FreeBSD.conf'), {'mode': None}), 26 | (("%(bsdploy_path)s/bootstrap-files/make.conf" % format_info, '/etc/make.conf'), {'mode': None}), 27 | (("%(bsdploy_path)s/bootstrap-files/pkg.conf" % format_info, '/usr/local/etc/pkg.conf'), {'mode': None})] 28 | run_mock.expected = [ 29 | ("su root -c '/tmp/enable_root_login_on_daemonology.sh'", {}, ''), 30 | ('rm /tmp/enable_root_login_on_daemonology.sh', {}, ''), 31 | ('mkdir -p "/usr/local/etc/pkg/repos"', {'shell': False}, ''), 32 | ('pkg update', {'shell': False}, ''), 33 | ('pkg install python27', {'shell': False}, '')] 34 | bootstrap() 35 | -------------------------------------------------------------------------------- /bsdploy/fabfile_daemonology.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | from bsdploy import bsdploy_path 3 | from bsdploy.bootstrap_utils import BootstrapUtils 4 | from fabric.api import env, put, run 5 | from os.path import join 6 | from time import sleep 7 | 8 | # a plain, default fabfile for jailhosts on ec2 9 | 10 | 11 | env.shell = '/bin/sh -c' 12 | 13 | 14 | def bootstrap(**kwargs): 15 | """ Bootstrap an EC2 instance that has been booted into an AMI from http://www.daemonology.net/freebsd-on-ec2/ 16 | Note: deprecated, current AMI images are basically pre-bootstrapped, they just need to be configured. 17 | """ 18 | # the user for the image is `ec2-user`, there is no sudo, but we can su to root w/o password 19 | original_host = env.host_string 20 | env.host_string = 'ec2-user@%s' % env.instance.uid 21 | bootstrap_files = env.instance.config.get('bootstrap-files', 'bootstrap-files') 22 | put('%s/authorized_keys' % bootstrap_files, '/tmp/authorized_keys') 23 | put(join(bsdploy_path, 'enable_root_login_on_daemonology.sh'), '/tmp/', mode='0775') 24 | run("""su root -c '/tmp/enable_root_login_on_daemonology.sh'""") 25 | # revert back to root 26 | env.host_string = original_host 27 | # give sshd a chance to restart 28 | sleep(2) 29 | run('rm /tmp/enable_root_login_on_daemonology.sh') 30 | 31 | # allow overwrites from the commandline 32 | env.instance.config.update(kwargs) 33 | 34 | bu = BootstrapUtils() 35 | bu.ssh_keys = None 36 | bu.upload_authorized_keys = False 37 | bu.bootstrap_files_yaml = 'daemonology-files.yml' 38 | bu.print_bootstrap_files() 39 | 40 | bu.create_bootstrap_directories() 41 | bu.upload_bootstrap_files({}) 42 | # we need to install python here, because there is no way to install it via 43 | # ansible playbooks 44 | bu.install_pkg('/', chroot=False, packages=['python27']) 45 | 46 | 47 | def fetch_assets(**kwargs): 48 | """ download bootstrap assets to control host. 49 | If present on the control host they will be uploaded to the target host during bootstrapping. 50 | """ 51 | # allow overwrites from the commandline 52 | env.instance.config.update(kwargs) 53 | BootstrapUtils().fetch_assets() 54 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../README.rst 2 | 3 | 4 | Dive in 5 | ------- 6 | 7 | .. toctree:: 8 | :maxdepth: 2 9 | 10 | installation 11 | quickstart 12 | 13 | 14 | Tutorial 15 | -------- 16 | 17 | A more in-depth tutorial than the quickstart. 18 | 19 | .. toctree:: 20 | :maxdepth: 2 21 | 22 | tutorial/overview 23 | tutorial/webserver 24 | tutorial/transmission 25 | tutorial/staging 26 | 27 | 28 | Setup 29 | ----- 30 | 31 | How to setup a host from scratch or make an existing one ready for BSDploy: 32 | 33 | .. toctree:: 34 | :maxdepth: 2 35 | 36 | setup/overview 37 | setup/provisioning-plain 38 | setup/provisioning-virtualbox 39 | setup/provisioning-ec2 40 | setup/bootstrapping 41 | setup/configuration 42 | 43 | 44 | General usage 45 | ------------- 46 | 47 | How to create and manage jails once the host is set up: 48 | 49 | .. toctree:: 50 | :maxdepth: 2 51 | 52 | usage/jails 53 | usage/ansible 54 | usage/fabric 55 | usage/ansible-with-fabric 56 | 57 | 58 | Special use cases 59 | ----------------- 60 | 61 | .. toctree:: 62 | :maxdepth: 2 63 | 64 | advanced/staging 65 | advanced/updating 66 | advanced/customizing-bootstrap 67 | 68 | 69 | Contribute 70 | ---------- 71 | 72 | Code and issues are hosted at github: 73 | 74 | - Issue Tracker: http://github.com/ployground/bsdploy/issues 75 | - Source Code: http://github.com/ployground/bsdploy 76 | - IRC: the developers can be found on #bsdploy on freenode.net 77 | 78 | 79 | License 80 | ------- 81 | 82 | The project is licensed under the Beerware license. 83 | 84 | 85 | TODO 86 | ---- 87 | 88 | The following features already exist but still need to be documented: 89 | 90 | - provisioning + bootstrapping 91 | - EC2 (daemonology based) 92 | - pre-configured SSH server keys 93 | - jail access 94 | - port forwarding 95 | - public IP 96 | - ZFS management 97 | - Creating and restoring ZFS snapshots 98 | - poudriere support 99 | - Upgrading strategies 100 | - 'vagrant mode' (use - virtualized - jails as development environment) 101 | 102 | The following features don't exist yet but should eventually :) 103 | 104 | - OS installers 105 | - homebrew 106 | - support vmware explicitly (like virtualbox)? 107 | -------------------------------------------------------------------------------- /bsdploy/roles/jails_host/templates/ezjail.conf: -------------------------------------------------------------------------------- 1 | # ezjail.conf - Example file, see ezjail.conf(5) 2 | # 3 | # Note: If you alter some of those variables AFTER creating your first 4 | # jail, you may have to adapt /etc/fstab.* and EZJAIL_PREFIX/etc/ezjail/* by 5 | # hand 6 | 7 | # Location of jail root directories 8 | # 9 | # Note: If you have spread your jails to multiple locations, use softlinks 10 | # to collect them in this directory 11 | ezjail_jaildir={{ jails_dir }} 12 | 13 | # Location of the tiny skeleton jail template 14 | # ezjail_jailtemplate=${ezjail_jaildir}/newjail 15 | 16 | # Location of the huge base jail 17 | # ezjail_jailbase=${ezjail_jaildir}/basejail 18 | 19 | # Location of your copy of FreeBSD's source tree 20 | # ezjail_sourcetree=/usr/src 21 | 22 | # This is where the install sub command defaults to fetch its packages from 23 | # ezjail_ftphost=ftp.freebsd.org 24 | 25 | # This is the command that is being executed by the console subcommand 26 | # ezjail_default_execute="/usr/bin/login -f root" 27 | 28 | # This is the flavour used by default when setting up a new jail 29 | # ezjail_default_flavour="" 30 | 31 | # This is the default location where ezjail archives its jails to 32 | # ezjail_archivedir="${ezjail_jaildir}/ezjail_archives" 33 | 34 | # base jail will provide a soft link from /usr/bin/perl to /usr/local/bin/perl 35 | # to accomodate all scripts using '#!/usr/bin/perl'... 36 | # ezjail_uglyperlhack="YES" 37 | 38 | # Default options for newly created jails 39 | # 40 | # Note: Be VERY careful about disabling ezjail_mount_enable. Mounting 41 | # basejail via nullfs depends on this. You will have to find other 42 | # ways to provide your jail with essential system files 43 | # ezjail_mount_enable="YES" 44 | # ezjail_devfs_enable="YES" 45 | # ezjail_devfs_ruleset="devfsrules_jail" 46 | ezjail_procfs_enable="NO" 47 | # ezjail_fdescfs_enable="YES" 48 | 49 | # ZFS options 50 | 51 | # Setting this to YES will start to manage the basejail and newjail in ZFS 52 | ezjail_use_zfs="YES" 53 | 54 | # Setting this to YES will manage ALL new jails in their own zfs 55 | # ezjail_use_zfs_for_jails="YES" 56 | 57 | # The name of the ZFS ezjail should create jails on, it will be mounted at the ezjail_jaildir 58 | 59 | ezjail_jailzfs="{{ ploy_jails_zfs_root }}" 60 | # ADVANCED, be very careful! 61 | # ezjail_zfs_properties="-o compression=lzjb -o atime=off" 62 | # ezjail_zfs_jail_properties="-o dedup=on" 63 | -------------------------------------------------------------------------------- /docs/tutorial/staging.rst: -------------------------------------------------------------------------------- 1 | Staging 2 | ======= 3 | 4 | So far we have developed our setup against a virtual machine which allowed us to experiment safely. 5 | Now it's time to deploy the setup in production. 6 | 7 | Principally, there are two possible approaches to this with ploy: 8 | 9 | - either define multiple hosts in one configuration or 10 | - use two separate configurations for each scenario 11 | 12 | In practice it's more practical to split the configuration, as it allows to re-use instances more easily. 13 | 14 | As a rule-of-thumb, if you ever need to configure two scenarios simultenously or either of the hosts need to communicate with each other, you should use a single configuration with multiple hosts (i.e. if you're configuring a cluster) but if you're setting up mutually exclusive, isolated scenarios such as staging vs. production you should use two different configurations. 15 | 16 | Another benefit of using two configurations is that you need to explicitly reference the non-default configuration which minimizes accidental modification of i.e. production hosts. 17 | 18 | In this tutorial we will use a second VirtualBox instances to simulate a production environment but in actual projects you would more likely define either plain instances or EC2 instances. 19 | 20 | To create a 'production' environment, create an additional configuration file ``etc/production.conf`` with the following contents:: 21 | 22 | [global] 23 | extends = ploy.conf 24 | 25 | [vb-instance:demo-production] 26 | vm-nic2 = nat 27 | vm-natpf2 = ssh,tcp,,44004,,22 28 | storage = 29 | --type dvddrive --medium ../downloads/mfsbsd-se-9.2-RELEASE-amd64.iso 30 | --medium vb-disk:boot 31 | 32 | [ez-master:jailhost] 33 | instance = demo-production 34 | 35 | Now we can start up the 'production' provider and run through the identical bootstrapping process as in the quickstart, except that we explicitly reference the production configuration:: 36 | 37 | ploy -c etc/production.conf start demo-production 38 | 39 | However, before we can bootstrap we need to import the ``bootstrap`` into our custom fabric file – this is because bootstrapping is internally implemented as a fabric task. 40 | 41 | Add the following import statement to your fabfile:: 42 | 43 | from bsdploy.fabfile_mfsbsd import bootstrap 44 | 45 | Now we can follow through:: 46 | 47 | ploy -c etc/production.conf bootstrap 48 | ploy -c etc/production.conf configure jailhost 49 | 50 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | import os 3 | 4 | here = os.path.abspath(os.path.dirname(__file__)) 5 | README = open(os.path.join(here, 'README.rst')).read() 6 | CHANGES = open(os.path.join(here, 'CHANGES.rst')).read() 7 | 8 | version = "3.0.0" 9 | 10 | install_requires = [ 11 | 'ansible;python_version>="3.10"', 12 | 'backports.lzma', 13 | 'PyYAML', 14 | 'jinja2', 15 | 'setuptools', 16 | 'ploy>=2.0.0', 17 | 'ploy_ansible>=2.0.0', 18 | 'ploy_ezjail>=2.0.0', 19 | 'ploy_fabric>=2.0.0', 20 | ] 21 | 22 | setup( 23 | name="bsdploy", 24 | version=version, 25 | description="A tool to remotely provision, configure and maintain FreeBSD jails", 26 | long_description=README + '\n\n\nChanges\n=======\n\n' + CHANGES, 27 | author='Tom Lazar', 28 | author_email='tom@tomster.org', 29 | maintainer='Florian Schulze', 30 | maintainer_email='mail@florian-schulze.net', 31 | url='http://github.com/ployground/bsdploy', 32 | include_package_data=True, 33 | classifiers=[ 34 | 'Development Status :: 4 - Beta', 35 | 'Environment :: Console', 36 | 'Intended Audience :: System Administrators', 37 | 'Operating System :: POSIX :: BSD :: FreeBSD', 38 | 'Programming Language :: Python :: 2.7', 39 | 'Programming Language :: Python :: 3.7', 40 | 'Programming Language :: Python :: 3.8', 41 | 'Programming Language :: Python :: 3.9', 42 | 'Programming Language :: Python :: 3.10', 43 | 'Topic :: System :: Installation/Setup', 44 | 'Topic :: System :: Systems Administration', 45 | ], 46 | license="BSD 3-Clause License", 47 | zip_safe=False, 48 | packages=['bsdploy'], 49 | install_requires=install_requires, 50 | python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, !=3.6.*', 51 | extras_require={ 52 | 'development': [ 53 | 'Sphinx', 54 | 'repoze.sphinx.autointerface', 55 | 'coverage', 56 | 'jarn.mkrelease', 57 | 'ploy_virtualbox>=2dev', 58 | 'pytest >= 2.4.2', 59 | 'pytest-flake8', 60 | 'tox', 61 | 'mock', 62 | ], 63 | }, 64 | entry_points=""" 65 | [console_scripts] 66 | ploy-download = bsdploy.download:run 67 | [ansible_paths] 68 | bsdploy = bsdploy:ansible_paths 69 | [ploy.plugins] 70 | bsdploy = bsdploy:plugin 71 | """) 72 | -------------------------------------------------------------------------------- /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: "CI" 3 | 4 | on: 5 | push: 6 | 7 | jobs: 8 | tests: 9 | name: "Python ${{ matrix.python-version }} (${{ matrix.tox-envs }})" 10 | runs-on: "ubuntu-latest" 11 | env: 12 | PY_COLORS: 1 13 | 14 | strategy: 15 | matrix: 16 | include: 17 | - python-version: "2.7" 18 | tox-envs: "py27" 19 | continue-on-error: false 20 | - python-version: "3.7" 21 | tox-envs: "py37" 22 | continue-on-error: false 23 | - python-version: "3.8" 24 | tox-envs: "py38" 25 | continue-on-error: false 26 | - python-version: "3.9" 27 | tox-envs: "py39" 28 | continue-on-error: false 29 | - python-version: "3.10" 30 | tox-envs: "py310" 31 | continue-on-error: false 32 | - python-version: "2.7" 33 | tox-envs: "py27-ansible19,py27-ansible24,py27-ansible25,py27-ansible26,py27-ansible27,py27-ansible28,py27-ansible29,py27-ansible210" 34 | continue-on-error: false 35 | - python-version: "3.7" 36 | tox-envs: "py37-ansible25,py37-ansible26,py37-ansible27,py37-ansible28,py37-ansible29,py37-ansible210" 37 | continue-on-error: false 38 | - python-version: "3.8" 39 | tox-envs: "py38-ansible25,py38-ansible26,py38-ansible27,py38-ansible28,py38-ansible29,py38-ansible210" 40 | continue-on-error: false 41 | - python-version: "3.9" 42 | tox-envs: "py39-ansible25,py39-ansible26,py39-ansible27,py39-ansible28,py39-ansible29,py39-ansible210" 43 | continue-on-error: false 44 | - python-version: "3.10" 45 | tox-envs: "py310-ansible4,py310-ansible5,py310-ansible6" 46 | continue-on-error: false 47 | 48 | steps: 49 | - uses: "actions/checkout@v2" 50 | - uses: "actions/setup-python@v2" 51 | with: 52 | python-version: "${{ matrix.python-version }}" 53 | - name: "Install dependencies" 54 | run: | 55 | set -xe -o nounset 56 | python -VV 57 | python -m site 58 | python -m pip install --upgrade pip setuptools wheel 59 | python -m pip install --upgrade virtualenv tox 60 | 61 | - name: "Run tox targets for ${{ matrix.python-version }}" 62 | continue-on-error: "${{ matrix.continue-on-error }}" 63 | run: | 64 | set -xe -o nounset 65 | python -m tox -a -vv 66 | python -m tox -v -e ${{ matrix.tox-envs }} -- -v --color=yes 67 | -------------------------------------------------------------------------------- /docs/advanced/customizing-bootstrap.rst: -------------------------------------------------------------------------------- 1 | Customizing bootstrap 2 | ===================== 3 | 4 | Currently the bootstrap API isn't ready for documentation and general use. 5 | In case you want to mess with it anyway, here are some things which will be safe to do. 6 | 7 | 8 | mfsBSD http proxy example 9 | ------------------------- 10 | 11 | If you are setting up many virtual machines for testing, then a caching http proxy to reduce the downloads comes in handy. 12 | You can achieve that by using `polipo `_ on the VM host and the following changes. 13 | 14 | First you need a custom fabfile: 15 | 16 | .. code-block:: python 17 | 18 | from bsdploy.fabfile_mfsbsd import _bootstrap, _mfsbsd 19 | from fabric.api import env, hide, run, settings 20 | from ploy.config import value_asbool 21 | 22 | 23 | def bootstrap(**kwargs): 24 | with _mfsbsd(env, kwargs): 25 | reboot = value_asbool(env.instance.config.get('bootstrap-reboot', 'true')) 26 | env.instance.config['bootstrap-reboot'] = False 27 | run('echo setenv http_proxy http://192.168.56.1:8123 >> /etc/csh.cshrc') 28 | run('echo http_proxy=http://192.168.56.1:8123 >> /etc/profile') 29 | run('echo export http_proxy >> /etc/profile') 30 | _bootstrap() 31 | run('echo setenv http_proxy http://192.168.56.1:8123 >> /mnt/etc/csh.cshrc') 32 | run('echo http_proxy=http://192.168.56.1:8123 >> /mnt/etc/profile') 33 | run('echo export http_proxy >> /mnt/etc/profile') 34 | if reboot: 35 | with settings(hide('warnings'), warn_only=True): 36 | run('reboot') 37 | 38 | For the ezjail initialization you have to add the following setting with a FreeBSD http mirror of your choice to your jail host config in ``ploy.conf``:: 39 | 40 | ansible-ploy_ezjail_install_host = http://ftp4.de.freebsd.org 41 | 42 | The ``_mfsbsd`` context manager takes care of setting the ``bootstrap-ssh-host-keys`` etc for mfsBSD. 43 | The ``_bootstrap`` function then runs the regular bootstrapping. 44 | 45 | For the jails you can use a startup script like this: 46 | 47 | .. code-block:: sh 48 | 49 | #!/bin/sh 50 | exec 1>/var/log/startup.log 2>&1 51 | chmod 0600 /var/log/startup.log 52 | set -e 53 | set -x 54 | echo setenv http_proxy http://192.168.56.1:8123 >> /etc/csh.cshrc 55 | echo http_proxy=http://192.168.56.1:8123 >> /etc/profile 56 | echo export http_proxy >> /etc/profile 57 | http_proxy=http://192.168.56.1:8123 58 | export http_proxy 59 | pkg install python27 60 | -------------------------------------------------------------------------------- /bsdploy/library/sysrc: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | DOCUMENTATION = ''' 3 | --- 4 | module: sysrc 5 | author: Florian Schulze 6 | short_description: Manage rc.conf via sysrc command 7 | description: 8 | - Manage rc.conf via sysrc command 9 | ''' 10 | 11 | 12 | class Sysrc(object): 13 | 14 | platform = 'FreeBSD' 15 | 16 | def __init__(self, module): 17 | self.module = module 18 | self.changed = False 19 | self.state = self.module.params.pop('state') 20 | self.name = self.module.params.pop('name') 21 | self.value = self.module.params.pop('value') 22 | self.dst = self.module.params.pop('dst') 23 | self.cmd = self.module.get_bin_path('sysrc', required=True) 24 | 25 | def sysrc(self, *params): 26 | params = list(params) 27 | if self.dst: 28 | params = ['-f', self.dst] + params 29 | return self.module.run_command([self.cmd] + params) 30 | 31 | def change_needed(self, *params): 32 | (rc, out, err) = self.sysrc('-c', *params) 33 | return rc != 0 34 | 35 | def change(self, *params): 36 | if self.module.check_mode: 37 | self.changed = True 38 | return 39 | (rc, out, err) = self.sysrc(*params) 40 | if rc != 0: 41 | self.module.fail_json(msg="Failed to run sysrc %s:\n%s" % (' '.join(params), err)) 42 | self.changed = True 43 | 44 | def __call__(self): 45 | result = dict(name=self.name, state=self.state) 46 | 47 | if self.state == 'present': 48 | if not self.value: 49 | self.module.fail_json(msg="When setting an rc.conf variable, a value is required.") 50 | setting = "%s=%s" % (self.name, self.value) 51 | if self.change_needed(setting): 52 | self.change(setting) 53 | elif self.state == 'absent': 54 | if self.change_needed('-x', self.name): 55 | self.change('-x', self.name) 56 | result['changed'] = self.changed 57 | return result 58 | 59 | 60 | MODULE_SPECS = dict( 61 | argument_spec=dict( 62 | name=dict(required=True, type='str'), 63 | value=dict(type='str'), 64 | state=dict(default='present', choices=['present', 'absent'], type='str'), 65 | dst=dict(type='str')), 66 | supports_check_mode=True) 67 | 68 | 69 | def main(): 70 | module = AnsibleModule(**MODULE_SPECS) 71 | result = Sysrc(module)() 72 | if 'failed' in result: 73 | module.fail_json(**result) 74 | else: 75 | module.exit_json(**result) 76 | 77 | 78 | from ansible.module_utils.basic import * 79 | if __name__ == "__main__": 80 | main() 81 | -------------------------------------------------------------------------------- /docs/usage/jails.rst: -------------------------------------------------------------------------------- 1 | Managing jails 2 | ============== 3 | 4 | The life cycle of a jail managed by BSDploy begins with an ``instance`` entry in ``ploy.conf``, i.e. like so:: 5 | 6 | [instance:webserver] 7 | ip = 10.0.0.1 8 | master = ploy-demo 9 | 10 | The minimally required parameters are the IP address of the jail (``ip``) and a reference to the jailhost (``master``) – the name of the jail is taken from the section name (in the example ``webserver``). 11 | 12 | .. note:: Unlike ``ez-master`` or other instances, names of jails are restricted by the constraints that FreeBSD imposes, namely they cannot contain dashes (``-``) 13 | 14 | BSDploy creates its own loopback device (``lo1``) during configuration and assigns a network of ``10.0.0.0/8`` by default (see ``bsdploy/roles/jails_host/defaults/main.yml`` for other values and their defaults), so you can use any ``10.x.x.x`` IP address out-of-the-box for your jails. 15 | 16 | Once defined, you can start the jail straight away. There is no explicit ``create`` command, if the jail does not exist during startup, it will be created on-demand:: 17 | 18 | # ploy start webserver 19 | INFO: Creating instance 'webserver' 20 | INFO: Starting instance 'webserver' with startup script, this can take a while. 21 | 22 | You can find out about the state of a jail by running ``ploy status JAILNAME``. 23 | 24 | A jail can be stopped with ``ploy stop JAILNAME``. 25 | 26 | A jail can be completely removed with ``ploy terminate JAILNAME``. This will destroy the ZFS filesystem specific to that jail. 27 | 28 | 29 | SSH Access 30 | ---------- 31 | 32 | BSDploy encourages jails to have a private IP address but compensates for that by providing convenient SSH access to them anyway, by automatically configuring an SSH ProxyCommand. 33 | 34 | Essentially, this means that you can SSH into any jail (or other instance) by providing it as a target for ploy's ``ssh`` command, i.e.:: 35 | 36 | # ploy ssh webserver 37 | FreeBSD 9.2-RELEASE (GENERIC) #6 r255896M: Wed Oct 9 01:45:07 CEST 2013 38 | 39 | Gehe nicht über Los. 40 | root@webserver:~ # 41 | 42 | Strictly speaking, you would need to address the jail instance together with the name of the host (to disambiguate multi-host scenarios) but since in this example there is only one jail host defined, ``webserver`` is enough, otherwise you would use ``jailhost-webserver``. 43 | 44 | 45 | rsync and scp 46 | ************* 47 | 48 | To access a jail with ``rsync`` (don't forget to install the ``rsync`` package into it!) or ``scp`` you can pass the ``ploy-ssh`` script into them like so:: 49 | 50 | scp -S ploy-ssh some.file webserver:/some/path/ 51 | rsync -e ploy-ssh some/path webserver:/some/path 52 | -------------------------------------------------------------------------------- /bsdploy/fabfile_digitalocean.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | from bsdploy.bootstrap_utils import BootstrapUtils 3 | from fabric.api import env, sudo, run, put, task 4 | from os import path 5 | from time import sleep 6 | 7 | # a plain, default fabfile for jailhosts on digital ocean 8 | 9 | 10 | env.shell = '/bin/sh -c' 11 | 12 | 13 | @task 14 | def bootstrap(**kwargs): 15 | """Digital Oceans FreeBSD droplets are pretty much already pre-bootstrapped, 16 | including having python2.7 and sudo etc. pre-installed. 17 | the only thing we need to change is to allow root to login (without a password) 18 | enable pf and ensure it is running 19 | """ 20 | 21 | bu = BootstrapUtils() 22 | # (temporarily) set the user to `freebsd` 23 | original_host = env.host_string 24 | env.host_string = 'freebsd@%s' % env.instance.uid 25 | # copy DO bsdclout-init results: 26 | if bu.os_release.startswith('10'): 27 | sudo("""cat /etc/rc.digitalocean.d/droplet.conf > /etc/rc.conf""") 28 | sudo("""sysrc zfs_enable=YES""") 29 | sudo("""sysrc sshd_enable=YES""") 30 | # enable and start pf 31 | sudo("""sysrc pf_enable=YES""") 32 | sudo("""sysrc -f /boot/loader.conf pfload=YES""") 33 | sudo('kldload pf', warn_only=True) 34 | sudo('''echo 'pass in all' > /etc/pf.conf''') 35 | sudo('''echo 'pass out all' >> /etc/pf.conf''') 36 | sudo('''chmod 644 /etc/pf.conf''') 37 | sudo('service pf start') 38 | # overwrite sshd_config, because the DO version only contains defaults 39 | # and a line explicitly forbidding root to log in 40 | sudo("""echo 'PermitRootLogin without-password' > /etc/ssh/sshd_config""") 41 | # additionally, make sure the root user is unlocked! 42 | sudo('pw unlock root') 43 | # overwrite the authorized keys for root, because DO creates its entries to explicitly 44 | # disallow root login 45 | bootstrap_files = env.instance.config.get('bootstrap-files', 'bootstrap-files') 46 | put(path.abspath(path.join(env['config_base'], bootstrap_files, 'authorized_keys')), '/tmp/authorized_keys', use_sudo=True) 47 | sudo('''mv /tmp/authorized_keys /root/.ssh/''') 48 | sudo('''chown root:wheel /root/.ssh/authorized_keys''') 49 | 50 | sudo("""service sshd fastreload""") 51 | # revert back to root 52 | env.host_string = original_host 53 | # give sshd a chance to restart 54 | sleep(2) 55 | # clean up DO cloudinit leftovers 56 | run("rm -f /etc/rc.d/digitalocean") 57 | run("rm -rf /etc/rc.digitalocean.d") 58 | run("rm -rf /usr/local/bsd-cloudinit/") 59 | run("pkg remove -y avahi-autoipd || true") 60 | 61 | # allow overwrites from the commandline 62 | env.instance.config.update(kwargs) 63 | 64 | bu.ssh_keys = None 65 | bu.upload_authorized_keys = False 66 | -------------------------------------------------------------------------------- /bsdploy/fabutils.py: -------------------------------------------------------------------------------- 1 | from fabric.api import env, local, run, sudo, task 2 | from fabric.contrib.project import rsync_project as _rsync_project 3 | from ploy.common import shjoin 4 | 5 | 6 | def rsync_project(*args, **kwargs): 7 | from bsdploy import log 8 | log.warning("rsync_project only works properly with direct ssh connections, you should use the rsync helper instead.") 9 | ssh_info = env.instance.init_ssh_key() 10 | ssh_info.pop('host') 11 | ssh_args = env.instance.ssh_args_from_info(ssh_info) 12 | kwargs['ssh_opts'] = '%s %s' % (kwargs.get('ssh_opts', ''), shjoin(ssh_args)) 13 | with env.instance.fabric(): 14 | env.host_string = "{user}@{host}".format( 15 | user=env.instance.config.get('user', 'root'), 16 | host=env.instance.config.get( 17 | 'host', env.instance.config.get( 18 | 'ip', env.instance.uid))) 19 | _rsync_project(*args, **kwargs) 20 | 21 | 22 | @task 23 | def rsync(*args, **kwargs): 24 | """ wrapper around the rsync command. 25 | 26 | the ssh connection arguments are set automatically. 27 | 28 | any args are just passed directly to rsync. 29 | you can use {host_string} in place of the server. 30 | 31 | the kwargs are passed on the 'local' fabric command. 32 | if not set, 'capture' is set to False. 33 | 34 | example usage: 35 | rsync('-pthrvz', "{host_string}:/some/src/directory", "some/destination/") 36 | """ 37 | kwargs.setdefault('capture', False) 38 | replacements = dict( 39 | host_string="{user}@{host}".format( 40 | user=env.instance.config.get('user', 'root'), 41 | host=env.instance.config.get( 42 | 'host', env.instance.config.get( 43 | 'ip', env.instance.uid)))) 44 | args = [x.format(**replacements) for x in args] 45 | ssh_info = env.instance.init_ssh_key() 46 | ssh_info.pop('host') 47 | ssh_info.pop('user') 48 | ssh_args = env.instance.ssh_args_from_info(ssh_info) 49 | cmd_parts = ['rsync'] 50 | cmd_parts.extend(['-e', "ssh %s" % shjoin(ssh_args)]) 51 | cmd_parts.extend(args) 52 | cmd = shjoin(cmd_parts) 53 | return local(cmd, **kwargs) 54 | 55 | 56 | @task 57 | def service(service=None, action='status'): 58 | if service is None: 59 | exit("You must provide a service name") 60 | sudo('service %s %s' % (service, action), warn_only=True) 61 | 62 | 63 | @task 64 | def pkg_upgrade(): 65 | run('pkg-static update') 66 | run('pkg-static install -U pkg') 67 | run('pkg update') 68 | run('pkg upgrade') 69 | print("Done.") 70 | 71 | 72 | @task 73 | def update_flavour_pkg(): 74 | """upgrade the pkg tool of the base flavour (so that newly created jails have the latest version)""" 75 | base_cmd = 'pkg-static -r /usr/jails/flavours/bsdploy_base' 76 | run('%s update' % base_cmd) 77 | run('%s install -U pkg' % base_cmd) 78 | run('%s update' % base_cmd) 79 | print("Done.") 80 | -------------------------------------------------------------------------------- /docs/usage/ansible-with-fabric.rst: -------------------------------------------------------------------------------- 1 | Combining Ansible and Fabric 2 | ============================ 3 | 4 | Both Ansible and Fabric are great tools, but they truly shine when used together. 5 | 6 | A common pattern for this is that a playbook is used to set up a state against which then a fabric script is (repeatedly) executed. 7 | 8 | For example an initial setup of an application, where a playbook takes care that all required directories are created with the proper permissions and into which then a fabric script performs an upload of the application code. 9 | 10 | 11 | Sharing variables between playbooks and fabric scripts 12 | ------------------------------------------------------ 13 | 14 | For such a collaboration both fabric and ansible need to know *where* all of this should take place, for instance. I.e. fabric has to know the location of the directory that the playbook has created. 15 | 16 | You can either define variables directly in ``ploy.conf`` or in group or host variables such as ``group_vars/all.yml`` or ``group_vars/webserver.yml``. 17 | 18 | To create key/value pairs in ``ploy.conf`` that are visible to ansible, you must prefix them with ``ansible-``. 19 | 20 | 21 | For example, you could create an entry in ploy.conf like so: 22 | 23 | .. code-block:: ini 24 | 25 | [instance:webserver] 26 | ... 27 | ansible-frontend_path = /opt/foo 28 | 29 | And then use the following snippet in a playbook: 30 | 31 | .. code-block:: yaml 32 | 33 | - name: ensure the www data directory exists 34 | file: path={{frontend_path}} state=directory mode=775 35 | 36 | Applying the playbook will then create the application directory as expected: 37 | 38 | .. code-block:: console 39 | 40 | ploy configure webserver 41 | PLAY [jailhost-webserver] ***************************************************** 42 | 43 | GATHERING FACTS *************************************************************** 44 | ok: [jailhost-webserver] 45 | 46 | TASK: [ensure the www data directory exists] ********************************** 47 | changed: [jailhost-webserver] 48 | 49 | Now let's create a fabric task that uploads the contents of that website:: 50 | 51 | def upload_website(): 52 | ansible_vars = fab.env.instance.get_ansible_variables() 53 | fab.put('dist/*', ansible_vars['frontend_path'] + '/') 54 | 55 | Notice, how we're accessing the ansible variables via Fabrics' ``env`` where ``ploy_fabrics`` has conveniently placed a ploy instance of our host. 56 | 57 | Let's run that: 58 | 59 | .. code-block:: console 60 | 61 | # ploy do webserver upload_website 62 | [root@jailhost-webserver] put: dist/index.html -> /opt/foo/index.html 63 | 64 | Putting variables that you want to share between fabric and ansible into your ``ploy.conf`` is the recommended way, as it upholds the configuration file as the canonical place for all of your configuration. 65 | 66 | However, until ploy learns how to deal with multi-line variable definitions, dealing with such requires setting them in ``group_vars/all.yml``. 67 | -------------------------------------------------------------------------------- /docs/usage/fabric.rst: -------------------------------------------------------------------------------- 1 | Fabric integration 2 | ================== 3 | 4 | .. epigraph:: 5 | 6 | `Fabric `_ is a Python library and command-line tool for streamlining the use of SSH for application deployment or systems administration tasks. 7 | 8 | BSDploy supports applying Fabric scripts to jails and jail hosts via the `ploy_fabric `_ plugin. 9 | 10 | There are two ways you can assign a fabric file to an instance: 11 | 12 | - *by convention* – if the project directory contains a directory with the same name as the instance, i.e. ``jailhost`` or ``jailhost-jailname`` containing a Python file named ``fabfile.py`` and no explicit file has been given, that file is used 13 | - by *explicit assigning one* in ``ploy.conf``, i.e.:: 14 | 15 | [instance:webserver] 16 | ... 17 | fabfile = ../fabfile.py 18 | 19 | 20 | Fabfile anatomy 21 | --------------- 22 | 23 | Let's take a look at an example ``fabfile.py``: 24 | 25 | .. code-block:: python 26 | :linenos: 27 | 28 | from fabric import api as fab 29 | 30 | fab.env.shell = '/bin/sh -c' 31 | 32 | def info(): 33 | fab.run('uname -a') 34 | 35 | def service(action='status'): 36 | fab.run('service nginx %s' % action) 37 | 38 | - (``1``) Fabric conveniently exposes all of its features in its ``api`` module, so we just import that for convenience 39 | - (``3``) Fabric assumes ``bash``, currently we must explicitly adapt it FreeBSD's default (this may eventually be handled by a future version of BSDploy) 40 | - (``8``) You can pass in parameters from the command line to a fabric task, see `Fabric's documentation on this `_ for more details. 41 | 42 | 43 | Fabfile execution 44 | ----------------- 45 | 46 | You can execute a task defined in that file by calling ``do``, i.e.:: 47 | 48 | # ploy do webserver service 49 | [root@jailhost-webserver] run: service nginx status 50 | [root@jailhost-webserver] out: nginx is running as pid 1563. 51 | 52 | Fabric has a `relatively obtuse syntax for passing in arguments `_ because it supports passing to multiple hosts in a single call. 53 | 54 | To alleviate this, ``ploy_fabric`` adds a simpler method in its ``do`` command, since that only always targets a single host. 55 | 56 | So, in our example, to restart the webserver you could do this:: 57 | 58 | # ploy do webserver service action=restart 59 | [root@jailhost-webserver] run: service nginx restart 60 | [root@jailhost-webserver] out: Performing sanity check on nginx configuration: 61 | [root@jailhost-webserver] out: nginx: the configuration file /usr/local/etc/nginx/nginx.conf syntax is ok 62 | [root@jailhost-webserver] out: nginx: configuration file /usr/local/etc/nginx/nginx.conf test is successful 63 | [root@jailhost-webserver] out: Stopping nginx. 64 | [root@jailhost-webserver] out: Waiting for PIDS: 1563. 65 | [root@jailhost-webserver] out: Performing sanity check on nginx configuration: 66 | [root@jailhost-webserver] out: nginx: the configuration file /usr/local/etc/nginx/nginx.conf syntax is ok 67 | [root@jailhost-webserver] out: nginx: configuration file /usr/local/etc/nginx/nginx.conf test is successful 68 | [root@jailhost-webserver] out: Starting nginx. 69 | 70 | 71 | You can also list all available tasks with the ``-l`` parameter:: 72 | 73 | ploy do webserver -l 74 | Available commands: 75 | 76 | info 77 | service 78 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | BSDploy – FreeBSD jail provisioning 2 | =================================== 3 | 4 | BSDploy is a comprehensive tool to remotely **provision**, **configure** and **maintain** `FreeBSD `_ `jail hosts and jails `_. 5 | 6 | Its main design goal is to lower the barrier to *repeatable jail setups*. 7 | 8 | Instead of performing updates on production hosts you are encouraged to update the *description* of your setup, test it against an identically configured staging scenario until it works as expected and then apply the updated configuration to production with confidence. 9 | 10 | 11 | Main Features 12 | ------------- 13 | 14 | - **provision** complete jail hosts from scratch 15 | 16 | - **describe** one or more jail hosts and their jails in a canonical configuration 17 | 18 | - **declarative configuration** – apply `Ansible `_ playbooks to hosts and jails 19 | 20 | - **imperative maintenance** – run `Fabric `_ scripts against hosts and jails 21 | 22 | - configure `ZFS pools and filesystems `_ with `whole-disk-encryption `_ 23 | 24 | - **modular provisioning** with plugins for `VirtualBox `_ and `Amazon EC2 `_ and an architecture to support more. 25 | 26 | 27 | How it works 28 | ------------ 29 | 30 | BSDploy takes the shape of a commandline tool by the name of ``ploy`` which is installed on a so-called *control host* (typically your laptop or desktop machine) with which you then control one or more *target hosts*. The only two things installed on target hosts by BSDploy are Python and ``ezjail`` – everything else stays on the control host. 31 | 32 | 33 | Example Session 34 | --------------- 35 | 36 | Here's what an abbreviated bootstrapping session of a simple website inside a jail on an Amazon EC2 instance could look like:: 37 | 38 | # ploy start ec-instance 39 | [...] 40 | # ploy configure jailhost 41 | [...] 42 | # ploy start webserver 43 | [...] 44 | # ploy configure webserver 45 | [...] 46 | # ploy do webserver upload_website 47 | 48 | 49 | Best of both worlds 50 | ------------------- 51 | 52 | Combining a declarative approach for setting up the initial state of a system with an imperative approach for providing maintenance operations on that state has significant advantages: 53 | 54 | 1. Since the imperative scripts have the luxury of running against a well-defined context, you can keep them short and concise without worrying about all those edge cases. 55 | 56 | 2. And since the playbooks needn't concern themselves with performing updates or other tasks you don't have to litter them with awkward states such as ``restarted`` or ``updated`` or – even worse – with non-states such as ``shell`` commands. 57 | 58 | 59 | Under the hood 60 | -------------- 61 | 62 | BSDploy's scope is quite ambitious, so naturally it does not attempt to do all of the work on its own. In fact, BSDPloy is just a fairly thin, slightly opinionated wrapper around existing excellent tools. 63 | 64 | In addition to the above mentioned Ansible and Fabric, it uses `ezjail `_ on the host to manage the jails and on the client numerous members of the `ployground family `_ for pretty much everything else. 65 | 66 | 67 | Full documentation 68 | ------------------ 69 | 70 | The full documentation is `hosted at RTD `_. 71 | -------------------------------------------------------------------------------- /docs/tutorial/transmission.rst: -------------------------------------------------------------------------------- 1 | Transmission 2 | ============ 3 | 4 | This part of the tutorial will demonstrate how to install and configure an existing web application. 5 | 6 | 7 | Using Ansible Roles 8 | ------------------- 9 | 10 | Unlike with the webserver from the previous example we *will* create a custom configuration, so instead of littering our top level directory with yet more playbooks and templates we will configure this instance using a role. 11 | 12 | Let's first create the required structure:: 13 | 14 | mkdir -p roles/transmission/tasks 15 | mkdir -p roles/transmission/templates 16 | mkdir -p roles/transmission/handlers 17 | 18 | Populate them with a settings template in ``roles/transmission/templates/settings.json``: 19 | 20 | .. code-block:: json 21 | 22 | { 23 | "alt-speed-up": 50, 24 | "alt-speed-down": 200, 25 | "speed-limit-down": 5000, 26 | "speed-limit-down-enabled": true, 27 | "speed-limit-up": 100, 28 | "speed-limit-up-enabled": true, 29 | "start-added-torrents": true, 30 | "trash-original-torrent-files": true, 31 | "watch-dir": "{{download_dir}}", 32 | "watch-dir-enabled": true, 33 | "rpc-whitelist": "127.0.0.*,10.0.*.*", 34 | "ratio-limit": 1.25, 35 | "ratio-limit-enabled": true 36 | } 37 | 38 | And in ``roles/transmission/handlers/main.yml``: 39 | 40 | .. code-block:: yaml 41 | 42 | --- 43 | - name: restart transmission 44 | service: name=transmission state=restarted 45 | 46 | And finally in ``roles/transmission/tasks/main.yml``: 47 | 48 | .. code-block:: yaml 49 | 50 | - name: Ensure helper packages are installed 51 | pkgng: name={{ item }} state=present 52 | with_items: 53 | - transmission-daemon 54 | - transmission-web 55 | - name: Setup transmission to start on boot 56 | service: name=transmission enabled=yes 57 | - name: Configure transmission 58 | template: src=settings.json dest=/usr/local/etc/transmission/home/settings.json backup=yes owner=transmission 59 | notify: 60 | - restart transmission 61 | 62 | The above tasks should look pretty familiar by now: 63 | 64 | - install the required packages (this time it's more than one and we demonstrate the ``with_items`` method) 65 | - enable it in ``rc.conf`` 66 | - Finally, as a new technique we upload a settings file as a template and... 67 | - ... use ansible's *handlers* to make sure that the service is reloaded every time we change its settings. 68 | 69 | 70 | Exercise One 71 | ------------ 72 | 73 | Publish the transmission daemon's web UI at ``http://192.168.56.100/transmission``. 74 | 75 | .. note:: Proxying to transmission can be a bit finicky as it requires certain CRSF protection headers, so here's a small spoiler/hint. 76 | 77 | This is the required nginx configuration to proxy to transmission:: 78 | 79 | location /transmission { 80 | proxy_http_version 1.1; 81 | proxy_set_header Connection ""; 82 | proxy_pass_header X-Transmission-Session-Id; 83 | proxy_pass http://transmissionweb; 84 | proxy_redirect off; 85 | proxy_buffering off; 86 | proxy_set_header Host $host; 87 | proxy_set_header X-Real-IP $remote_addr; 88 | } 89 | 90 | 91 | Exercise Two 92 | ------------ 93 | 94 | Publish the downloads directory via nginx so users can download finished torrents from ``http://192.168.56.100/downloads``. 95 | 96 | Do this by configuring an additional jail that has read-only access to the download directory and publishes using its own nginx which is then targetted by the webserver jail. 97 | -------------------------------------------------------------------------------- /docs/setup/overview.rst: -------------------------------------------------------------------------------- 1 | Overview 2 | ======== 3 | 4 | The life cycle of a system managed by BSDploy is divided into three distinct segments: 5 | 6 | 1. provisioning 7 | 8 | 2. bootstrapping 9 | 10 | 3. configuration 11 | 12 | Specifically: 13 | 14 | - At the end of the provisioning process we have a system that has booted into an installer (usually mfsBSD) which is ready for bootstrapping 15 | - At the end of the bootstrapping process we have installed and booted into a vanilla FreeBSD system with just the essential requirements to complete the actual configuration 16 | - At the end of the configuration process we have a FreeBSD system with a pre-configured ``ezjail`` setup which can be managed by BSDploy. 17 | 18 | Conceptually, the provider of a jailhost is a separate entity from the jailhost itself, i.e. in a VirtualBox based setup you could have the following:: 19 | 20 | [vb-instance:ploy-demo] 21 | vm-ostype = FreeBSD_64 22 | [...] 23 | 24 | [ez-master:jailhost] 25 | instance = ploy-demo 26 | [...] 27 | 28 | [ez-instance:webserver] 29 | master = jailhost 30 | [...] 31 | 32 | Here we define a VirtualBox instance (``vb-instance``) named ``ploy-demo`` and a so-called master named ``jailhost`` which in turn contains a jail instance named ``webserver``. 33 | 34 | This approach allows us to hide the specific details of a provider and also to replace it with another one, i.e. :doc:`in a staging scenario ` where a production provider such as a 'real' server or EC2 instance is replaced with a VirtualBox. 35 | 36 | In the following we will document the entire provisioning process for each supported provider separately (eventhough there is a large amount of overlap), so you can simply pick the one that suits your setup and then continue with the configuration step which is then the same for all providers. 37 | 38 | In a nutshell, though, given the previous example setup, what you need to do is this:: 39 | 40 | ploy start ploy-demo 41 | ploy bootstrap jailhost 42 | ploy configure jailhost 43 | ploy start webserver 44 | 45 | 46 | Project setup and directory structure 47 | ------------------------------------- 48 | 49 | ``ploy`` and thus by extension also BSDploy have a "project based" approach, meaning that all configuration, playbooks, assets etc. for one given use case (a.k.a. "project") are contained in a single directory, as opposed to having a central configuration such as ``/usr/local/etc/ploy.conf``. 50 | 51 | The minimal structure of such a directory is to contain a subdirectory named ``etc`` in which the main configuration file ``ploy.conf`` is located. 52 | 53 | Since BSDploy treats this directory technically as an :ref:`Ansible directory ` you typically would also have additional top-level directories such as ``roles``, ``host_vars`` etc. 54 | 55 | Another (entirely optional) convention is to have a top-level directory named ``downloads`` where assets such as installer images or package archives are placed. 56 | 57 | This approach was also chosen because most of the times project directories are version controlled and for example ``downloads/`` can then be safely ignored, because all of its contents are a) large, binary files and b) easily replaceable whereas ``etc/`` often contains sensitive, project specific data such as private keys, certificates etc. and may be even part of a different repository altogether. 58 | 59 | 60 | Next step: Provisioning 61 | ----------------------- 62 | 63 | Basically, unless you want to use one of the specific provisioners such as EC2 or VirtualBox, just use a plain instance: 64 | 65 | .. toctree:: 66 | :maxdepth: 1 67 | 68 | provisioning-plain 69 | provisioning-virtualbox 70 | provisioning-ec2 71 | -------------------------------------------------------------------------------- /docs/usage/ansible.rst: -------------------------------------------------------------------------------- 1 | Ansible integration 2 | =================== 3 | 4 | Not only does BSDploy use `ansible playbooks `_ internally to configure jail hosts (via the `ploy_ansible `_ plugin), but you can use it to configure jails, too, of course. 5 | 6 | .. note:: An important difference between using ansible *as is* and via BSDploy is that unlike ansible, BSDploy does not require (or indeed support) what ansible refers to as an `inventory file `_ – all hosts are defined within ploy's configuration file. 7 | 8 | You can either assign roles or a playbook file to an instance and then apply them via the ``configure`` command. 9 | 10 | 11 | Playbook assignment 12 | ------------------- 13 | 14 | There are two ways you can assign a playbook to an instance – either via naming convention or by using the ``playbook`` parameter. 15 | 16 | Assigning by convention works by creating a top-level ``*.yml`` file with the same name as the instance you want to assign it to. 17 | 18 | Keep in mind, however, that unlike when targetting instances from the command line, there is no aliassing in effect, meaning you *must* use the *full* name of the jail instance in the form of *hostname*-*jailname*, i.e. in our example ``jailhost-webserver.yml``. 19 | 20 | The *by-convention* method is great for quick one-off custom tasks for an instance – often in the playbook you include one or more roles that perform the bulk of the work and add a few top-level tasks in the playbook that didn't warrant the overhead of a a role of their own. However, the target of the playbook is effectively hard-coded in the name of the file, so re-use is not possible. 21 | 22 | If you want to use the same playbook for multiple instances, you can assign it by using ``playbook = PATH``, where ``PATH`` usually is a top level ``*.yml`` file. 23 | 24 | Note that any paths are always relative to the location of the ``ploy.conf`` file, so usually you would have to refer to a top-level file like so:: 25 | 26 | [instance:webserver1] 27 | ... 28 | playbook = ../nginx.yml 29 | 30 | [instance:webserver2] 31 | ... 32 | playbook = ../nginx.yml 33 | 34 | 35 | Role assignment 36 | --------------- 37 | 38 | Playbook assignment can be convenient when dealing with ad-hoc or one-off tasks but both ansible and BSDploy strongly encourage the use of `roles `_. 39 | 40 | Role assignment works just as you've probably guessed it by now: by using a ``roles`` parameter. Unlike with ``playbook`` you can assign more than one role. You do this by listing one role per line, indented for legibility, i.e.:: 41 | 42 | [instance:webserver] 43 | roles = 44 | nginx 45 | haproxy 46 | zfs-snapshots 47 | 48 | .. note:: Assignment of roles and assignment of playbooks are mutually exclusive. If you try to do both, BSDploy will raise an error. 49 | 50 | 51 | Tags 52 | ---- 53 | 54 | When applying a playbook or roles via the ``configure`` command you can select only certain tags of them to be executed by adding the ``-t`` parameter, i.e. like so:: 55 | 56 | ploy configure webserver -t config 57 | 58 | To select multiple tags, pass them in comma-separated. Note that in this case you must make sure you don't add any whitespace, i.e.:: 59 | 60 | ploy configure webserver -t config,cert 61 | 62 | 63 | .. _dir-structure: 64 | 65 | Directory structure 66 | ------------------- 67 | 68 | The directory of a BSDploy environment is also an `ansible project structure `_ meaning, you can create and use top-level directories such as ``roles``, ``group_vars`` or even ``library``, etc. (see the link about the `ansible directory structure `_). 69 | -------------------------------------------------------------------------------- /docs/setup/provisioning-virtualbox.rst: -------------------------------------------------------------------------------- 1 | Provisioning VirtualBox instances 2 | ================================= 3 | 4 | BSDploy provides automated provisioning of `VirtualBox `_ instances via the `ploy_virtualbox plugin `_. 5 | 6 | .. Note:: The plugin is not installed by default when installing BSDploy, so you need to install it additionally like so ``pip install ploy_virtualbox``. 7 | 8 | 9 | Unlike with :doc:`plain instances ` the configuration doesn't just describe existing instances but is used to create them. Consider the following entry in ``ploy.conf``:: 10 | 11 | [vb-instance:ploy-demo] 12 | vm-nic2 = nat 13 | vm-natpf1 = ssh,tcp,,44003,,22 14 | storage = 15 | --medium vb-disk:defaultdisk 16 | --type dvddrive --medium http://mfsbsd.vx.sk/files/iso/10/amd64/mfsbsd-se-10.3-RELEASE-amd64.iso --medium_sha1 564758b0dfebcabfa407491c9b7c4b6a09d9603e 17 | 18 | 19 | VirtualBox instances are configured using the ``vb-instance`` prefix and you can set parameters of the virtual machine by prefixing them with ``vm-``. For additional details on which parameters are available and what they mean, refer to `the plugin's documentation `_ and the documentation of the VirtualBox commandline tool `VBoxManage `_, in particualar for `VBoxManage createvm `_ and `VBoxManage modifyvm `_. 20 | 21 | Having said that, BSDploy provides a number of convenience defaults for each instance, so in most cases you won't need much more than in the above example. 22 | 23 | 24 | Default hostonly network 25 | ------------------------ 26 | 27 | Unless you configure otherwise, BSDploy will tell VirtualBox to 28 | 29 | - create a host-only network interface named ``vboxnet0`` 30 | - assign the first network interface to that 31 | - create a DHCP server for the address range ``192.168.56.100-254`` 32 | 33 | This means that a) during bootstrap the VM will receive a DHCP address from that range but more importantly b) you are free to assign your own static IPs from the range *below* (i.e. ``192.168.56.10``) because the existence of the VirtualBox DHCP server will ensure that that IP is reachable from the host system. This allows you to assign known, good static IP addresses to all of your VirtualBox instances. 34 | 35 | 36 | Default disk setup 37 | ------------------ 38 | 39 | As you can see in the example above, there is a reference to a disk named ``defaultdisk`` in the ``storage`` parameter of the ``vb-instance`` entry. If you reference a disk of that name, BSDploy will automatically provision a virtual sparse disk of 100Gb size. In practice it's often best to leave that assignment in place (it's where the OS will be installed onto during bootstrap) and instead configure additional disks for data storage. 40 | 41 | 42 | Boot images 43 | ----------- 44 | 45 | Also note, that we reference a mfsBSD boot image above and assign it to the optical drive. By providing an external URL with a checksum, ``ploy_virtualbox`` will download that image for us (by default into ``~/.ploy/downloads/``) and connect it to the instance. 46 | 47 | 48 | First Startup 49 | ------------- 50 | 51 | Unlike ``VBoxManage`` BSDploy does not provide an explicit *create* command, instead just start the instance and if it doesn't exist already, BSDploy will create it for you on-demand:: 52 | 53 | ploy start ploy-demo 54 | 55 | Since the network interface is configured via DHCP, we cannot know under which IP the VM will be available. Instead the above snippet configures portforwarding, so regardless of the IP it gets via DHCP, we will access the VM via SSH using the host ``localhost`` and (in the example) port ``44003``. Adjust these values to your needs and use them during :doc:`/setup/bootstrapping`. 56 | 57 | .. Note:: In addtion to starting a VM you can also use the ``stop`` and ``terminate`` commands.. 58 | -------------------------------------------------------------------------------- /docs/installation.rst: -------------------------------------------------------------------------------- 1 | Client requirements 2 | =================== 3 | 4 | BSDploy and its dependencies are written in `Python `_ and thus should run on pretty much any platform, although it's currently only been tested on Mac OS X and FreeBSD. 5 | 6 | 7 | Server requirements 8 | =================== 9 | 10 | .. warning:: 11 | BSDploy is intended for initial configuration of a jail host before any jails have been installed. 12 | While technically possible, BSDploy is not intended for managing existing systems with non-BSDploy jails. 13 | Running against hosts that have not been bootstrapped by BSDploy can result in loss of data. 14 | 15 | A FreeBSD system that wants to be managed by BSDploy will 16 | 17 | - need to have `ezjail `_ installed 18 | - as well as `Python `_ 19 | - must have SSH access enabled (either for root or with ``sudo`` configured). 20 | - have ZFS support (BSDploy does not support running on non-ZFS filesystems) 21 | 22 | Strictly speaking, BSDploy only needs Python for the initial configuration of the jailhost. If you chose to perform that step yourself or use a pre-existing host, you won't need Python on the host, just ezjail. 23 | 24 | Normally, BSDploy will take care of these requirements for you during :doc:`bootstrapping ` but in situations where this is not possible, manually providing the abovementioned requirements should allow you to :doc:`apply BSDploy's host configuration ` anyway. 25 | 26 | BSDploy supports FreeBSD >= 9.2, including 10.3. 27 | 28 | 29 | Client Installation 30 | =================== 31 | 32 | Since BSDploy is still early in development, there is currently only a packaged version for FreeBSD but no others such as i.e. homebrew, aptitude etc.) available yet. 33 | 34 | You can however install beta releases from PyPI or a bleeding edge development version from github. 35 | 36 | 37 | Installing on FreeBSD 38 | --------------------- 39 | 40 | BSDploy is available from FreeBSD ports as ``sysutils/bsdploy``, for details `check it at FreshPorts `_. 41 | 42 | 43 | Installing from PyPI 44 | -------------------- 45 | 46 | BSDploy and its dependencies are written in Python, so you can install them from the official Python Packaging Index (a.k.a. PyPI). 47 | 48 | The short version: 49 | 50 | .. code-block:: console 51 | :linenos: 52 | 53 | virtualenv . 54 | source bin/activate 55 | pip install bsdploy 56 | 57 | (``1``) BSDploy has specific requirements in regards to Fabric and ansible (meaning, their latest version will not neccessarily work with the latest version of BSDploy until the latter is adjusted) it is therefore strongly recommended to install BSDploy into its own virtualenv. 58 | 59 | To do so, you will need Python and virtualenv installed on your system, i.e. 60 | 61 | - on **Mac OS X** using ``homebrew`` you would install ``brew install pyenv-virtualenv``. 62 | - on **FreeBSD** using ``pkg`` you would ``pkg install py27-virtualenv`` 63 | 64 | (``2``) To use the version installed inside this virtualenv it is suggested to 'source' the python interpreter. This will add the ``bin`` directory of the virtualenv (temporarily) to your ``$PATH`` so you can use the binaries installed inside it just as if they were installed globally. Note, that the default ``activate`` works for bash, if you're using ``tcsh`` (the default on FreeBSD you will have to ``source bin/activate.csh``) 65 | 66 | 67 | Installing from github 68 | ---------------------- 69 | 70 | To follow along the latest version of BSDploy you need Python and virtualenv plus – obviously – ``git``. Then:: 71 | 72 | git clone https://github.com/ployground/bsdploy.git 73 | cd bsdploy 74 | make 75 | 76 | This will check out copies of BSDploy's immediate dependencies (``ploy`` and friends) and create the ploy* executables inside ``bin``. You can either add the ``bin`` directory to your path or symlink them into somewhere that's already on your path, but as described above, it is recommended to source the ``virtualenv`` to have a 'global' installation of BSDploy:: 77 | 78 | source bin/activate 79 | 80 | When keeping your checkout up-to-date it is usually a good idea to update the ``ploy`` packages (located inside ``src``), as well. The best way to do so is to use the provided ``develop`` command after updating the bsdploy repository itself like so:: 81 | 82 | git pull 83 | bin/develop up -v 84 | 85 | The ``-v`` flag will show any git messages that arise during the update. 86 | -------------------------------------------------------------------------------- /docs/setup/configuration.rst: -------------------------------------------------------------------------------- 1 | Configuring a jailhost 2 | ====================== 3 | 4 | Once the host has been successfully bootstrapped, we are left with a vanilla FreeBSD installation with the following exceptions: 5 | 6 | - we have key based SSH access as root 7 | - Python has been installed 8 | 9 | But before we can create and manage jails, a few tasks still remain, in particular 10 | 11 | - installation and configuration of ``ezjail`` 12 | - ZFS setup and layout, including optional encryption 13 | - jail specific network setup 14 | 15 | Unlike bootstrapping, this final step is implemented using ansible playbooks and has been divided into multiple roles, so all that is left for us is to apply the ``configure`` to the ``ez-master`` instance, i.e. like so:: 16 | 17 | ploy configure ploy-demo 18 | 19 | Among other things, this will create an additional zpool named ``tank`` (by default) which will be used to contain the jails. 20 | 21 | 22 | Configuring as non-root 23 | ----------------------- 24 | 25 | While bootstrapping currently *must* be performed as ``root`` (due to the fact that mfsBSD itself requires root login) some users may not want to enable root login for their systems. 26 | 27 | If you want to manage a jailhost with a non-root user, you must perform the following steps manually: 28 | 29 | - install ``sudo`` on the jailhost 30 | - create a user account and enable SSH access for it 31 | - enable passwordless ``sudo`` access for it 32 | - disable SSH login for root (currently, automatically enabled during bootstrapping) 33 | 34 | Additionally, you *must* configure the system using a playbook (i.e. simply assigning one or more roles won't work in this case) and in that playbook you must set the username and enable ``sudo``, i.e. like so: 35 | 36 | .. code-block:: yaml 37 | 38 | --- 39 | - hosts: jailhost 40 | user: tomster 41 | sudo: yes 42 | roles: 43 | # apply the built-in bsdploy role jails_host 44 | - jails_host 45 | 46 | And, of course, once bootstrapped, you need to set the same username in ``ploy.conf``: 47 | 48 | .. code-block:: ini 49 | 50 | [ez-master:jailhost] 51 | user = tomster 52 | 53 | 54 | Full-Disk encryption with GELI 55 | ------------------------------ 56 | 57 | One of the many nice features of FreeBSD is its `modular, layered handling of disks (GEOM) `_. 58 | This allows to inject a crypto layer into your disk setup without that higher up levels (such as ZFS) need to be aware of it, which is exactly what BSDploy supports. 59 | 60 | If you add ``bootstrap-geli = yes`` to an ``ez-master`` entry, BSDploy will generate a passphrase, encrypt the GEOM provider for the ``tank`` zpool and write the passphrasse to ``'/root/geli-passphrase`` and configures the appropriate ``geli_*_flag`` entries in ``rc.conf`` so that it is used automatically during booting. 61 | 62 | The upshot is that when enabling GELI you still will have the same convenience as without encryption but can easily up the ante by removing the passphrase file (remember to keep it safe, though!). You will, however, need to attach the device manually after the system has booted and enter the passphrase. 63 | 64 | 65 | Additional host roles 66 | --------------------- 67 | 68 | 69 | The main bulk of work has been factored into the role ``jails_host`` which also is the default role. 70 | 71 | If the network of your host is configured via DHCP you can apply an additional role named ``dhcp_host`` which takes care of the hosts sshd configuration when the DHCP lease changes. 72 | To have it applied when calling ``configure`` add an explicit ``roles`` parameter to your ``ez-instance``: 73 | 74 | .. code-block:: ini 75 | :emphasize-lines: 3-5 76 | 77 | [ez-master:ploy-demo] 78 | [...] 79 | roles = 80 | dhcp_host 81 | jails_host 82 | [...] 83 | 84 | Technically, BSDploy injects its own roles to ansibles playbook path, so to apply your own custom additions, add additional roles to a top-level directory named ``roles`` and include them in your configuration and they will be applied as well. 85 | 86 | Common tasks for such additional setup could be setting up a custom ZFS layout, configuring snapshots and backups, custom firewall rules etc, basically anything that you would not want to lock inside a jail. 87 | 88 | 89 | .. note:: Curently, the ``jails_host`` playbook is rather monolithic, but given the way ansible works, there is the possibility of making it more granular, i.e. by tagging and/or parametrisizing specific sub-tasks and then to allow applying tags and parameter values in ``ploy.conf``. 90 | -------------------------------------------------------------------------------- /docs/setup/provisioning-plain.rst: -------------------------------------------------------------------------------- 1 | Provisioning plain instances 2 | ============================ 3 | 4 | The most simple provider simply assumes an already existing host. Here any configuration is purely descriptive. Unlike with the other providers we tell BSDploy how things are and it doesn't do anything about it. 5 | 6 | For example:: 7 | 8 | [plain-instance:ploy-demo] 9 | host = 10.0.1.2 10 | 11 | At the very least you will need to provide a ``host`` (IP address or hostname). 12 | 13 | Additionally, you can provide a non-default SSH ``port`` and a ``user`` to connect with (default is ``root``). 14 | 15 | 16 | Local hardware 17 | -------------- 18 | 19 | The most common scenario for using a plain instance is physical hardware that you have access to and can boot from a custom installer medium. 20 | 21 | 22 | Download installer image 23 | ************************ 24 | 25 | First, you need to download the installer image and copy it onto a suitable medium, i.e. an USB stick. 26 | 27 | As mentioned in the quickstart, BSDploy uses `mfsBSD `_ which is basically the official FreeBSD installer plus pre-configured SSH access. Also, it provides a small cross-platform helper for downloading assets via HTTP which also checks the integrity of the downloaded file:: 28 | 29 | mkdir downloads 30 | ploy-download http://mfsbsd.vx.sk/files/images/9/amd64/mfsbsd-se-9.2-RELEASE-amd64.img 9f354d30fe5859106c1cae9c334ea40852cb24aa downloads/ 31 | 32 | 33 | Creating a bootable USB medium (Mac OSX) 34 | **************************************** 35 | 36 | For the time being we only provide instructions for Mac OS X, sorry! If you run Linux you probably already know how to do this, anyway :-) 37 | 38 | - Run ``diskutil list`` to see which drives are currently in your system. 39 | - insert your medium 40 | - re-run ``diskutil list`` and notice which number it has been assigned (N) 41 | - run ``diskutil unmountDisk /dev/diskN`` 42 | - run ```sudo dd if=downloads/mfsbsd-se-9.2-RELEASE-amd64.img of=/dev/diskN bs=1m`` 43 | - run ``diskutil unmountDisk /dev/diskN`` 44 | - now you can remove the stick and boot the target host from it 45 | 46 | 47 | Booting into mfsBSD 48 | ******************* 49 | 50 | Insert the USB stick into the *target host* and boot from it. Log in as ``root`` using the pre-configured password ``mfsroot``. Either note the name of the ethernet interface and the IP address it has been given by running ``ifconfig`` and (temporarily) set it in ``ploy.conf`` or configure them one-time to match your expectations. 51 | 52 | 53 | One-Time manual network configuration 54 | +++++++++++++++++++++++++++++++++++++ 55 | 56 | BSDploy needs to access the installer via SSH and it in turn will need to download assets from the internet during installation, so we need to configure the interface, the gateway, DNS and sshd. 57 | 58 | In the above mentioned example that could be:: 59 | 60 | ifconfig em0 netmask 255.255.255.0 61 | route add default 10.0.1.1 62 | 63 | Since usually, the router also performs as DNS you would edit ``/etc/resolv.conf`` to look like so:: 64 | 65 | nameserver 10.0.1.1 66 | 67 | Finally restart sshd:: 68 | 69 | service sshd restart 70 | 71 | To verify that all is ready, ssh into the host as user ``root`` with the password ``mfsroot``. Once that works, log out again and you are ready to continue with :doc:`/setup/bootstrapping`. 72 | 73 | 74 | Hetzner 75 | ------- 76 | 77 | The German ISP `Hetzner `_ provides dedicated servers with FreeBSD support. In a nutshell, boot the machine into their so-called *Rescue System* `using their robot `_ and choose *FreeBSD* as OS. The machine will boot into a modified version of mfsBSD. 78 | 79 | The web UI will then provide you with a one-time root password – make sure it works by SSHing into the host as ``root`` and you are ready for continuing with :doc:`/setup/bootstrapping`. 80 | 81 | 82 | vmWare 83 | ------ 84 | 85 | Since BSDploy (currently) doesn't support automated provisioning of vmWare instances (like it does for VirtualBox) you will need to manually create a vmWare instance and then follow the steps above for it, except that instead of downloading the image referenced there you need one specifically for booting into a virtual machine, IOW download like so:: 86 | 87 | mkdir downloads 88 | ploy-download http://mfsbsd.vx.sk/files/iso/9/amd64/mfsbsd-se-9.2-RELEASE-amd64.iso 4ef70dfd7b5255e36f2f7e1a5292c7a05019c8ce downloads/ 89 | 90 | Then create a new virtual machine, set the above image as boot device and continue with :doc:`/setup/bootstrapping`. 91 | -------------------------------------------------------------------------------- /docs/advanced/staging.rst: -------------------------------------------------------------------------------- 1 | Staging 2 | ======= 3 | 4 | One use case that BSDploy was developed for is to manage virtual copies of production servers. 5 | 6 | The idea is to have a safe environment that mirrors the production environment as closely as possible into which new versions of applications or system packages can be installed or database migrations be tested etc. 7 | 8 | But safe testing and experimenting is just one benefit, the real benefit comes into play, once you've tweaked your staging environment to your liking. Since BSDploy makes it easy for you to capture those changes and tweaks into playbooks and fabric scripts, applying them to production is now just a matter changing the target of a ``ploy`` command. 9 | 10 | 11 | Extended configuration files 12 | **************************** 13 | 14 | To help you keep the staging and production environment as similar as possible we will use the ability of ``ploy`` to inherit configuration files. 15 | 16 | We define the general environment (one or more jail hosts and jails) in a ``base.conf`` and create two 'top-level' configuration files ``staging.conf`` and ``production.conf`` which each extend ``base.conf``. 17 | 18 | During deployment we specify the top-level configuration file via' ``ploy -c xxx.conf``. A variation of this is to name the staging configuration file ``ploy.conf`` which then acts as default. This has the advantage that during development of the environment you needn't bother with explicitly providing a configuration file and that when moving to production you need to make the extra, explicit step of (now) providing a configuration file, thus minimizing the danger of accidentally deploying something onto production. 19 | 20 | Here is an example ``base.conf``:: 21 | 22 | [ez-master:jailhost] 23 | instance = provisioner 24 | roles = jails_host 25 | 26 | [instance:webserver] 27 | master = jailhost 28 | ip = 10.0.0.1 29 | 30 | 31 | [instance:appserver] 32 | master = jailhost 33 | ip = 10.0.0.2 34 | 35 | and ``staging.conf``:: 36 | 37 | [global] 38 | extends = base.conf 39 | 40 | [vb-instance:provisioner] 41 | vm-ostype = FreeBSD_64 42 | [...] 43 | 44 | and ``production.conf``:: 45 | 46 | [global] 47 | extends = base.conf 48 | 49 | [ec2-instance:provisioning] 50 | ip = xxx.xxx.xxx.xxx 51 | instance_type = m1.small 52 | # FreeBSD 9.2-RELEASE instance-store eu-west-1 from daemonology.net/freebsd-on-ec2/ 53 | image = ami-3e1ef949 54 | [...] 55 | 56 | .. note:: In practice, it can often be useful to split this out even further. We have made good experiences with using a virtualbox based setup for testing the deployment during its development, then once that's finished we apply it to a public server (that can be accessed by stakeholders of the project for evaluation) that actually runs on the same platform as the production machine and once that has been approved we finally apply it to the production environment. YMMV. 57 | 58 | 59 | Staging with FQDN 60 | ***************** 61 | 62 | A special consideration when deploying web applications is how to test the entire stack, including the webserver with fully qualified domain names and SSL certificates etc. 63 | 64 | BSDploy offers a neat solution using the following three components: 65 | 66 | - template based webserver configuration 67 | - `xip.io `_ based URLs for testing 68 | - VirtualBox `host-only networking `_ 69 | 70 | IOW if you're deploying a fancy-pants web application at your production site ``fancypants.com`` using i.e. a nginx configuration snippet like such:: 71 | 72 | server { 73 | server_name fancypants.com; 74 | [...] 75 | } 76 | 77 | Change it to this:: 78 | 79 | server { 80 | server_name fancypants.com{{fqdn_suffix}}; 81 | [...] 82 | } 83 | 84 | And in your ``staging.conf`` you define ``fqdn_suffix`` to be i.e. ``.192.168.56.10.xip.io`` and in ``production.conf`` to an empty string. 85 | 86 | Finally, configure the VirtualBox instance in staging to use a second nic (in addition to the default host-only interface) via DHCP so it can access the internet:: 87 | 88 | [vb-instance:provisioner] 89 | vm-nic2 = nat 90 | 91 | ``ploy_virtualbox`` will ensure that the virtual network ``vboxnet0`` exists (if it doesn't already). 92 | You can then use the fact that VirtualBox will set up a local network (default is ``192.168.56.xxx``) with a DHCP range from ``.100 - .200`` and assign your nic1 (``em0`` in our case) a static IP of, i.e. ``192.168.56.10`` which you then can use in the abovementioned xip.io domain name. 93 | 94 | The net result? Deploy to staging and test your web application's full stack (including https, rewriting etc.) in any browser under ``https://fancypants.com.192.168.56.10.xip.io`` in the knowledge that the only difference between that setup and your (eventual) production environment is a single suffix string. 95 | -------------------------------------------------------------------------------- /bsdploy/tests/conftest.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | from mock import Mock 3 | import pytest 4 | 5 | 6 | def pytest_addoption(parser): 7 | parser.addoption( 8 | "--quickstart-bsdploy", help="Run the quickstart with this bsdploy sdist", 9 | action="store", dest="quickstart_bsdploy", default=False) 10 | parser.addoption( 11 | "--ansible-version", help="The ansible version to use for quickstart tests, defaults to newest", 12 | action="store", dest="ansible_version") 13 | 14 | 15 | @pytest.fixture 16 | def default_mounts(): 17 | return '\n'.join([ 18 | '/dev/md0 on / (ufs, local, read-only)', 19 | 'devfs on /dev (devfs, local, multilabel)', 20 | 'tmpfs on /rw (tmpfs, local)', 21 | 'devfs on /mnt/dev (devfs, local, multilabel)']) 22 | 23 | 24 | @pytest.fixture 25 | def ctrl(ployconf, tempdir): 26 | from ploy import Controller 27 | import ploy.tests.dummy_plugin 28 | ctrl = Controller(tempdir.directory) 29 | ctrl.plugins = { 30 | 'dummy': ploy.tests.dummy_plugin.plugin} 31 | ctrl.configfile = ployconf.path 32 | return ctrl 33 | 34 | 35 | @pytest.fixture 36 | def fabric_integration(): 37 | from ploy_fabric import _fabric_integration 38 | # this needs to be done before any other fabric module import 39 | _fabric_integration.patch() 40 | 41 | 42 | class RunResult(str): 43 | pass 44 | 45 | 46 | @pytest.fixture 47 | def run_result(): 48 | def run_result(out, rc): 49 | result = RunResult(out) 50 | result.return_code = rc 51 | result.succeeded = rc == 0 52 | result.failed = rc != 0 53 | return result 54 | return run_result 55 | 56 | 57 | @pytest.fixture 58 | def run_mock(fabric_integration, monkeypatch): 59 | run = Mock() 60 | 61 | def _run(command, **kwargs): 62 | try: 63 | expected = run.expected.pop(0) 64 | except IndexError: # pragma: nocover 65 | expected = '', '', '' 66 | cmd, kw, result = expected 67 | assert command == cmd 68 | assert kwargs == kw 69 | return result 70 | 71 | run.side_effect = _run 72 | run.expected = [] 73 | monkeypatch.setattr('bsdploy.bootstrap_utils.run', run) 74 | monkeypatch.setattr('fabric.contrib.files.run', run) 75 | return run 76 | 77 | 78 | @pytest.fixture 79 | def put_mock(fabric_integration, monkeypatch): 80 | put = Mock() 81 | 82 | def _put(*args, **kw): 83 | try: 84 | expected = put.expected.pop(0) 85 | except IndexError: # pragma: nocover 86 | expected = ((), {}) 87 | eargs, ekw = expected 88 | assert len(args) == len(eargs) 89 | for arg, earg in zip(args, eargs): 90 | if earg is object: 91 | continue 92 | if hasattr(arg, 'name'): 93 | assert arg.name == earg 94 | else: 95 | assert arg == earg 96 | assert sorted(kw.keys()) == sorted(ekw.keys()) 97 | for k in kw: 98 | if ekw[k] is object: 99 | continue 100 | assert kw[k] == ekw[k], "kw['%s'](%r) != ekw['%s'](%r)" % (k, kw[k], k, ekw[k]) 101 | 102 | put.side_effect = _put 103 | put.expected = [] 104 | monkeypatch.setattr('bsdploy.bootstrap_utils.put', put) 105 | monkeypatch.setattr('fabric.contrib.files.put', put) 106 | return put 107 | 108 | 109 | @pytest.fixture 110 | def local_mock(fabric_integration, monkeypatch): 111 | from mock import Mock 112 | local = Mock() 113 | 114 | def _local(command, **kwargs): 115 | try: 116 | expected = local.expected.pop(0) 117 | except IndexError: # pragma: nocover 118 | expected = '', '', '' 119 | cmd, kw, result = expected 120 | assert command == cmd 121 | assert kwargs == kw 122 | return result 123 | 124 | local.side_effect = _local 125 | local.expected = [] 126 | monkeypatch.setattr('bsdploy.bootstrap_utils.local', local) 127 | return local 128 | 129 | 130 | @pytest.fixture 131 | def env_mock(ctrl, fabric_integration, monkeypatch, ployconf): 132 | from fabric.api import env 133 | ployconf.fill([ 134 | '[dummy-instance:test_instance]']) 135 | env.instance = ctrl.instances['test_instance'] 136 | return env 137 | 138 | 139 | @pytest.fixture 140 | def environ_mock(monkeypatch): 141 | environ = {} 142 | monkeypatch.setattr('os.environ', environ) 143 | return environ 144 | 145 | 146 | @pytest.fixture 147 | def yesno_mock(monkeypatch): 148 | yesno = Mock() 149 | 150 | def _yesno(question): 151 | try: 152 | expected = yesno.expected.pop(0) 153 | except IndexError: # pragma: nocover 154 | expected = '', False 155 | cmd, result = expected 156 | assert question == cmd 157 | print(question) 158 | return result 159 | 160 | yesno.side_effect = _yesno 161 | yesno.expected = [] 162 | monkeypatch.setattr('bsdploy.bootstrap_utils.yesno', yesno) 163 | monkeypatch.setattr('ploy.common.yesno', yesno) 164 | return yesno 165 | -------------------------------------------------------------------------------- /bsdploy/roles/jails_host/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: bind host sshd to primary ip 3 | lineinfile: 4 | dest: /etc/ssh/sshd_config 5 | regexp: ^ListenAddress 6 | line: 'ListenAddress {{ ploy_jail_host_sshd_listenaddress }}' 7 | notify: restart sshd 8 | 9 | - { include: ntpd.yml, tags: ntpd } 10 | - { include: obsolete-ipf.yml, tags: pf } 11 | - { include: pf.yml, tags: pf } 12 | 13 | - name: Setup cloned interfaces 14 | sysrc: 15 | name: cloned_interfaces 16 | value: "{{ ploy_jail_host_cloned_interfaces }}" 17 | notify: restart network 18 | 19 | - meta: flush_handlers 20 | 21 | # The sysctl module in ansible adds spaces around the equal sign in 22 | # /etc/sysctl.conf which breaks in FreeBSD 10, so we do this manually 23 | - name: Enable security.jail.allow_raw_sockets 24 | lineinfile: 25 | dest: /etc/sysctl.conf 26 | regexp: ^security.jail.allow_raw_sockets\s*= 27 | line: security.jail.allow_raw_sockets=1 28 | notify: reload sysctl 29 | tags: sysctl 30 | - name: Enable security.jail.sysvipc_allowed 31 | lineinfile: 32 | dest: /etc/sysctl.conf 33 | regexp: ^security.jail.sysvipc_allowed\s*= 34 | line: security.jail.sysvipc_allowed=1 35 | notify: reload sysctl 36 | tags: sysctl 37 | 38 | - name: Ensure helper packages are installed (using http proxy) 39 | pkgng: 40 | name: "ezjail" 41 | state: "present" 42 | when: ploy_http_proxy is defined 43 | environment: 44 | http_proxy: "{{ploy_http_proxy}}" 45 | https_proxy: "{{ploy_http_proxy}}" 46 | 47 | - name: Ensure helper packages are installed 48 | pkgng: 49 | name: "ezjail" 50 | state: "present" 51 | when: ploy_http_proxy is not defined 52 | 53 | - name: Set default jail interface 54 | sysrc: 55 | name: jail_interface 56 | value: "{{ ploy_jail_host_default_jail_interface }}" 57 | notify: restart ezjail 58 | - name: Set default jail parameters 59 | sysrc: 60 | name: jail_parameters 61 | value: "allow.raw_sockets=1 allow.sysvipc=1" 62 | notify: restart ezjail 63 | - name: Set default jail exec stop 64 | sysrc: 65 | name: jail_exec_stop 66 | value: "/bin/sh /etc/rc.shutdown" 67 | notify: restart ezjail 68 | - name: Enable jail_parallel_start 69 | sysrc: 70 | name: jail_parallel_start 71 | value: "YES" 72 | 73 | - name: Enable ezjail in rc.conf 74 | service: 75 | name: ezjail 76 | enabled: yes 77 | notify: restart ezjail 78 | 79 | - name: Setup ezjail.conf 80 | template: src=ezjail.conf dest=/usr/local/etc/ezjail.conf 81 | notify: restart ezjail 82 | 83 | - name: Setup data zpool 84 | zpool: 85 | name: "{{ ploy_bootstrap_data_pool_name }}" 86 | geli: "{{ ploy_bootstrap_geli|bool }}" 87 | version: "{{ ploy_bootstrap_zpool_version }}" 88 | devices: "{{ ploy_bootstrap_data_pool_devices }}" 89 | raid_mode: "{{ ploy_bootstrap_raid_mode }}" 90 | 91 | - name: Set data zpool options 92 | zfs: 93 | name="{{ ploy_bootstrap_data_pool_name }}" 94 | state=present 95 | atime=off 96 | checksum=fletcher4 97 | 98 | - name: Jails ZFS file system 99 | zfs: 100 | name="{{ ploy_jails_zfs_root }}" 101 | state=present 102 | mountpoint=/usr/jails 103 | 104 | - name: Initialize ezjail (using http proxy) 105 | command: "ezjail-admin install -h {{ ploy_ezjail_install_host }} -r {{ ploy_ezjail_install_release|default(ansible_distribution_release) }} creates=/usr/jails/basejail" 106 | when: ploy_http_proxy is defined 107 | environment: 108 | http_proxy: "{{ploy_http_proxy}}" 109 | https_proxy: "{{ploy_http_proxy}}" 110 | 111 | - name: Initialize ezjail (may take a while) 112 | command: "ezjail-admin install -h {{ ploy_ezjail_install_host }} -r {{ ploy_ezjail_install_release|default(ansible_distribution_release) }} creates=/usr/jails/basejail" 113 | when: ploy_http_proxy is not defined 114 | 115 | - name: Create pkg cache folder 116 | file: dest=/var/cache/pkg/All/ state=directory owner=root group=wheel 117 | 118 | - name: Directory for jail flavour "bsdploy_base" 119 | file: dest=/usr/jails/flavours/bsdploy_base state=directory owner=root group=wheel 120 | 121 | - name: The .ssh directory for root in bsdploy_base flavour 122 | file: 123 | dest: "/usr/jails/flavours/bsdploy_base/{{ploy_root_home_path}}/.ssh" 124 | state: directory 125 | mode: "0600" 126 | owner: "{{ploy_root_user_name}}" 127 | group: wheel 128 | 129 | - name: The etc directory in bsdploy_base flavour 130 | file: dest=/usr/jails/flavours/bsdploy_base/etc state=directory owner=root group=wheel 131 | - name: The etc/ssh directory in bsdploy_base flavour 132 | file: dest=/usr/jails/flavours/bsdploy_base/etc/ssh state=directory owner=root group=wheel 133 | 134 | - name: /etc/make.conf in bsdploy_base flavour 135 | copy: src=make.conf dest=/usr/jails/flavours/bsdploy_base/etc/make.conf owner=root group=wheel 136 | - name: /usr/local/etc/pkg/repos directory in bsdploy_base flavour 137 | file: dest=/usr/jails/flavours/bsdploy_base/usr/local/etc/pkg/repos state=directory owner=root group=wheel 138 | - name: /usr/local/etc/pkg.conf in bsdploy_base flavour 139 | template: src=pkg.conf dest=/usr/jails/flavours/bsdploy_base/usr/local/etc/pkg.conf owner=root group=wheel 140 | - name: /usr/local/etc/pkg/repos/FreeBSD.conf in bsdploy_base flavour 141 | template: src=FreeBSD.conf dest=/usr/jails/flavours/bsdploy_base/usr/local/etc/pkg/repos/FreeBSD.conf owner=root group=wheel 142 | 143 | 144 | # 145 | # configure bsdploy_base flavour 146 | # 147 | 148 | - name: rc.conf for bsdploy_base flavour 149 | copy: src=base_flavour_rc.conf dest=/usr/jails/flavours/bsdploy_base/etc/rc.conf owner=root group=wheel 150 | - name: sshd_config for bsdploy_base flavour 151 | copy: src=base_flavour_sshd_config dest=/usr/jails/flavours/bsdploy_base/etc/ssh/sshd_config owner=root group=wheel 152 | - name: motd for bsdploy_base flavour 153 | copy: src=base_flavour_motd dest=/usr/jails/flavours/bsdploy_base/etc/motd owner=root group=wheel 154 | - name: copy some settings from host to bsdploy_base flavour 155 | shell: cmp -s {{ item.src }} {{ item.dest }} || cp -v {{ item.src }} {{ item.dest }} 156 | register: _cp_settings_result 157 | changed_when: _cp_settings_result.stdout|default() != '' 158 | with_items: 159 | - { src: "/etc/resolv.conf", dest: "/usr/jails/flavours/bsdploy_base/etc/resolv.conf" } 160 | - { src: "/{{ploy_root_home_path}}/.ssh/authorized_keys", dest: "/usr/jails/flavours/bsdploy_base{{ploy_root_home_path}}/.ssh/authorized_keys" } 161 | -------------------------------------------------------------------------------- /bsdploy/tests/test_roles.py: -------------------------------------------------------------------------------- 1 | from bsdploy import bsdploy_path 2 | import os 3 | import pytest 4 | 5 | 6 | @pytest.fixture 7 | def ctrl(ployconf, tempdir): 8 | from ploy import Controller 9 | import bsdploy 10 | import ploy_ezjail 11 | import ploy_ansible 12 | ployconf.fill([ 13 | '[ez-master:jailhost]', 14 | 'host = jailhost']) 15 | ctrl = Controller(configpath=ployconf.directory) 16 | ctrl.plugins = { 17 | 'bsdploy': bsdploy.plugin, 18 | 'ezjail': ploy_ezjail.plugin, 19 | 'ansible': ploy_ansible.plugin} 20 | ctrl.configfile = ployconf.path 21 | return ctrl 22 | 23 | 24 | def get_all_roles(): 25 | roles_path = os.path.join(bsdploy_path, 'roles') 26 | roles = [] 27 | for item in os.listdir(roles_path): 28 | if os.path.isdir(os.path.join(roles_path, item)): 29 | roles.append(item) 30 | return sorted(roles) 31 | 32 | 33 | def _iter_tasks(block): 34 | from ansible.playbook.block import Block 35 | for task in block: 36 | if isinstance(task, Block): 37 | for task in _iter_tasks(task.block): 38 | yield task 39 | else: 40 | yield task 41 | 42 | 43 | def iter_tasks(plays): 44 | from ploy_ansible import ANSIBLE1 45 | if ANSIBLE1: 46 | for play in plays: 47 | for task in play.tasks(): 48 | if task.meta: 49 | if task.meta == 'flush_handlers': # pragma: nocover - branch coverage only on failure 50 | continue 51 | raise ValueError # pragma: nocover - only on failure 52 | yield play, task 53 | else: 54 | for play in plays: 55 | for task in _iter_tasks(play.compile()): 56 | if task.action == 'meta': 57 | meta_action = task.args.get('_raw_params') 58 | if meta_action == 'flush_handlers': # pragma: nocover - branch coverage only on failure 59 | continue 60 | if meta_action == 'role_complete': # pragma: nocover - branch coverage only on failure 61 | continue 62 | raise ValueError # pragma: nocover - only on failure 63 | yield play, task 64 | 65 | 66 | def get_plays(pb, monkeypatch): 67 | from ploy_ansible import ANSIBLE1 68 | if ANSIBLE1: 69 | plays = [] 70 | monkeypatch.setattr('ansible.playbook.PlayBook._run_play', plays.append) 71 | pb.run() 72 | return plays 73 | else: 74 | return pb.get_plays() 75 | 76 | 77 | def test_roles(ctrl, monkeypatch): 78 | instance = ctrl.instances['jailhost'] 79 | pb = instance.get_playbook() 80 | plays = get_plays(pb, monkeypatch) 81 | tasks = [] 82 | for play, task in iter_tasks(plays): 83 | tasks.append(task.name) 84 | assert tasks == [ 85 | 'bind host sshd to primary ip', 86 | 'Enable ntpd in rc.conf', 87 | 'Disable public use of ntpd', 88 | 'Check for old ipnat_rules setting', 89 | 'Remove ipfilter from rc.conf', 90 | 'Remove ipfilter_rules from rc.conf', 91 | 'Remove ipmon from rc.conf', 92 | 'Remove ipmon_flags from rc.conf', 93 | 'Remove ipnat from rc.conf', 94 | 'Remove ipnat_rules from rc.conf', 95 | 'Enable pf in rc.conf', 96 | 'Check for /etc/pf.conf', 97 | 'Default pf.conf', 98 | 'Stat of /dev/pf', 99 | 'Checking pf', 100 | 'Setup pf.conf', 101 | 'Reload pf.conf', 102 | 'Enable gateway in rc.conf', 103 | 'Setup cloned interfaces', 104 | 'Enable security.jail.allow_raw_sockets', 105 | 'Enable security.jail.sysvipc_allowed', 106 | 'Ensure helper packages are installed (using http proxy)', 107 | 'Ensure helper packages are installed', 108 | 'Set default jail interface', 109 | 'Set default jail parameters', 110 | 'Set default jail exec stop', 111 | 'Enable jail_parallel_start', 112 | 'Enable ezjail in rc.conf', 113 | 'Setup ezjail.conf', 114 | 'Setup data zpool', 115 | 'Set data zpool options', 116 | 'Jails ZFS file system', 117 | 'Initialize ezjail (using http proxy)', 118 | 'Initialize ezjail (may take a while)', 119 | 'Create pkg cache folder', 120 | 'Directory for jail flavour "bsdploy_base"', 121 | 'The .ssh directory for root in bsdploy_base flavour', 122 | 'The etc directory in bsdploy_base flavour', 123 | 'The etc/ssh directory in bsdploy_base flavour', 124 | '/etc/make.conf in bsdploy_base flavour', 125 | '/usr/local/etc/pkg/repos directory in bsdploy_base flavour', 126 | '/usr/local/etc/pkg.conf in bsdploy_base flavour', 127 | '/usr/local/etc/pkg/repos/FreeBSD.conf in bsdploy_base flavour', 128 | 'rc.conf for bsdploy_base flavour', 129 | 'sshd_config for bsdploy_base flavour', 130 | 'motd for bsdploy_base flavour', 131 | 'copy some settings from host to bsdploy_base flavour'] 132 | 133 | 134 | def test_all_role_templates_tested(ctrl, monkeypatch, request): 135 | instance = ctrl.instances['jailhost'] 136 | instance.config['roles'] = ' '.join(get_all_roles()) 137 | pb = instance.get_playbook() 138 | plays = get_plays(pb, monkeypatch) 139 | # import after running to avoid module import issues 140 | from bsdploy.tests import test_templates 141 | templates = [] 142 | for play, task in iter_tasks(plays): 143 | if task.action != 'template': 144 | continue 145 | loader = play.get_loader() 146 | src = task.args.get('src') 147 | template_path = loader.path_dwim_relative( 148 | task._role._role_path, 'templates', src) 149 | if not os.path.exists(template_path): # pragma: nocover - only on failure 150 | raise ValueError 151 | name = src.lower() 152 | for rep in ('-', '.'): 153 | name = name.replace(rep, '_') 154 | templates.append(( 155 | name, 156 | dict( 157 | path=task._role._role_path, 158 | role_name=task._role.get_name(), 159 | name=src, task_name=task.name))) 160 | test_names = [x for x in dir(test_templates) if x.startswith('test_')] 161 | for name, info in templates: 162 | test_name = 'test_%s_%s' % (info['role_name'], name) 163 | if not any(x for x in test_names if x.startswith(test_name)): # pragma: nocover - only on failure 164 | pytest.fail("No test '{0}' for template '{name}' of task '{task_name}' in role '{role_name}' at '{path}'.".format(test_name, **info)) 165 | -------------------------------------------------------------------------------- /CHANGES.rst: -------------------------------------------------------------------------------- 1 | 3.0.0 - 2022-08-17 2 | ================== 3 | 4 | - [feature] support Python 3.10. 5 | 6 | 7 | 3.0.0b4 - 2020-09-08 8 | ==================== 9 | 10 | - [feature] support ``bootstrap-password`` option. 11 | - [feature] allow override of ``destroygeom`` via ``bootstrap-destroygeom``. 12 | - [feature] allow override of packages installed during bootstrap via ``bootstrap-packages``. 13 | - [fix] correct path to devfs device in mfsbsd boostrap script. 14 | 15 | 16 | 3.0.0b3 - 2019-06-09 17 | ==================== 18 | 19 | - [feature] Python 3.x support with Ansible >= 2.4.x. 20 | - [feature] the sysrc module supports ``dst`` option to use another file then the default ``/etc/rc.conf``. 21 | - [change] renamed ``bootstrap-host-keys`` to ``bootstrap-ssh-host-keys``. 22 | - [change] reintroduce ``bootstrap-ssh-fingerprints`` to allow overriding of ``ssh-fingerprints`` for bootstrapping. 23 | 24 | 25 | 3.0.0b2 - 2018-02-11 26 | ==================== 27 | 28 | - [change] ask before automatically generating missing ssh host keys during bootstrap. 29 | - [change] the default location for ``bootstrap-files`` changed from ``[playbooks-directory]/bootstrap-files`` to ``[playbooks-directory]/[instance-uid]/bootstrap-files``. 30 | - [change] renamed ``firstboot-update`` to ``bootstrap-firstboot-update`` to match the other variables. 31 | 32 | 33 | 3.0.0b1 - 2018-02-07 34 | ==================== 35 | 36 | - [change] switch to use ploy 2.0.0 and Ansible 2.4.x. 37 | - [feature] the ``fabfile`` option is set if ``[instance-name]/fabfile.py`` exists when the more specific ``[master-name]-[instance-name]/fabfile.py`` doesn't exist. 38 | 39 | - [fix]: honour the ``boottrap-packages`` setting for mfsbsd. 40 | 41 | 42 | 2.3.0 - 2017-11-13 43 | ================== 44 | 45 | - [fix] fix pf round-robin lockups. thanks to @igalic for reporting and fixing this issue 46 | - [feature] add ed25519 support in bootstrap needed for paramiko>=2. you should check whether you have ``ssh_host_ed25519_key*`` files on your host which you might want to copy to your bootstrap files directory alongside the other ``ssh_host_*_key*`` files 47 | - [change] removed local rsa1 host key generation 48 | 49 | 50 | 2.2.0 - 2016-11-08 51 | ================== 52 | 53 | - [feature] add fabric helpers to keep pkg up-to-date on the host, inside jails and for the bsdploy flavour 54 | - [feature] add support for bootstrapping on Digital Ocean by setting `bootstrap` to `digitalocean` in the `ez-master` definition 55 | - [fix] allow setting a non-default zfs root for ezjail by setting `jails_zfs_root` in the `ez-master` definition 56 | 57 | 58 | 2.1.0 - 2015-07-26 59 | ================== 60 | 61 | - [feature] enable jail_parallel_start in rc.conf of jail host 62 | - [fix] import existing zpool in ``zpool`` ansible module if the name matches 63 | - [fix] try to attach geli device first in ``zpool`` ansible module, in case it already exists, only if that fails create it from scratch 64 | - [fix] properly handle multiple geli encrypted devices in ``zpool`` ansible module 65 | - [fix] also honor the ``ploy_jail_host_pkg_repository`` variable during bootstrapping (not just jailhost configuration) 66 | - [feature] files copied during bootstrap can be encrypted using the ``ploy vault`` commands. This is useful for the private ssh host keys in ``bootstrap-files``. 67 | - [fix] fixed setting of virtualbox defaults, so they can be properly overwritten 68 | - [feature] added new variables: ploy_jail_host_cloned_interfaces/ploy_jail_host_default_jail_interface to give more flexiblity around network interface setup 69 | - [change] dropped support for Ansible versions < 1.8 (supports 1.8.x and 1.9.x now) 70 | - [fix] honour proxy setting while installing ezjail itself, not just during ezjail's install run (thanks mzs114! https://github.com/ployground/bsdploy/pull/81) 71 | 72 | 73 | 2.0.0 - 2015-03-05 74 | ================== 75 | 76 | - [feature] add support for http proxies 77 | - [change] deactivate pkg's *auto update* feature by default 78 | - [feature] add support for `firstboot-freebsd-update `_ (disabled by default) 79 | - [change] [BACKWARDS INCOMPATIBLE] switched from ipfilter to pf - you must convert any existing ``ipnat_rules`` to the new ``pf_nat_rules``. 80 | - [feature] provide defaults for VirtualBox instances (less boilerplate) 81 | - [fix] set full /etc/ntp.conf instead of trying to fiddle with an existing one. 82 | - [feature] Support configuration as non-root user (see https://github.com/ployground/bsdploy/issues/62) 83 | - [change] switched to semantic versioning (see http://semver.org) 84 | 85 | 86 | 1.3 - 2014-11-28 87 | ================ 88 | 89 | - [deprecation] rsync_project is not working in all cases, print a warning 90 | - [feature] added rsync helper, which is a tiny wrapper around the rsync command 91 | - [fix] change format of /usr/local/etc/pkg/repos/FreeBSD.conf so the package 92 | repository is properly recognized 93 | - [change] use quarterly package repository everywhere 94 | 95 | 96 | 1.2 - 2014-10-26 97 | ================ 98 | 99 | - [feature] provide default and by-convention assignment of fabfiles 100 | - [doc] document provisioning of EC2 instances 101 | - [fix] fix string escapes for geli setup in rc.conf 102 | - [feature] make sshd listen address configurable 103 | - [fix] fix permission of periodic scripts in zfs_auto_snapshot role 104 | - [doc] describe how to use a http proxy for mfsBSD 105 | 106 | 107 | 1.1.1 - 2014-09-25 108 | ================== 109 | 110 | - increase memory for virtual machines in documentation from 512MB to 1024MB 111 | - fix escaping for jail settings in rc.conf preventing jails from starting 112 | 113 | 114 | 1.1.0 - 2014-08-13 115 | ================== 116 | 117 | - use FreeBSD 10.0 as default for bootstrapping and documentation 118 | - always encode result of templates as utf-8 119 | - fix compatibility with ansible 1.7 120 | 121 | 122 | 1.0.0 - 2014-07-20 123 | ================== 124 | 125 | - added bsdploy.fabutils with a wrapper for rsync_project 126 | - automatically set env.shell for fabric scripts. 127 | - generate ssh host keys locally during bootstrap if possible. 128 | - set ``fingerprint`` option for ezjail master automatically if a ssh host key exists locally. 129 | 130 | 131 | 1.0b4 - 2014-07-08 132 | ================== 133 | 134 | - remove custom ``ploy`` and ``ploy-ssh`` console scripts. 135 | 136 | 137 | 1.0b3 - 2014-07-07 138 | ================== 139 | 140 | - make ``ploy_virtualbox`` an optional dependency 141 | 142 | 143 | 1.0b2 - 2014-07-07 144 | ================== 145 | 146 | - migrate from ``mr.awsome*`` dependencies to ``ploy*`` 147 | - various bugfixes 148 | - added tests 149 | 150 | 151 | 1.0b1 - 2014-06-17 152 | ================== 153 | 154 | - Initial public release 155 | -------------------------------------------------------------------------------- /docs/setup/provisioning-ec2.rst: -------------------------------------------------------------------------------- 1 | Provisioning Amazon EC2 instances 2 | ================================= 3 | 4 | BSDploy provides automated provisioning of `Amazon EC2 `_ instances via the `ploy_ec2 plugin `_. 5 | 6 | .. Note:: The plugin is not installed by default when installing BSDploy, so you need to install it additionally like so ``pip install ploy_ec2``. 7 | 8 | Like with :doc:`Virtualbox instances ` the configuration doesn't just describe existing instances but is used to create them. 9 | 10 | The first step is always to tell ploy where to find your AWS credentials in ``ploy.conf``:: 11 | 12 | [ec2-master:default] 13 | access-key-id = ~/.aws/foo.id 14 | secret-access-key = ~/.aws/foo.key 15 | 16 | Then, to define EC2 instances use the ``ec2-`` prefix, for example: 17 | 18 | .. code-block:: console 19 | :linenos: 20 | 21 | [ec2-instance:production-backend] 22 | region = eu-west-1 23 | placement = eu-west-1a 24 | keypair = xxx 25 | ip = xxx.xxx.xxx.xxx 26 | instance_type = c3.xlarge 27 | securitygroups = production 28 | image = ami-c847b2bf 29 | user = root 30 | startup_script = ../startup_script 31 | 32 | Let's go through this briefly, the full details are available at the `ploy_ec2 documentation `_: 33 | 34 | - ``2-3`` here you set the region and placement where the instance shall reside 35 | - ``4`` you can optionally provide a keypair (that you must create or upload to EC2 beforehand). If you do so, the key will be used to grant you access to the newly created machine. See the section below regarding the ``startup_script``. 36 | - ``5`` if you have an Elastic IP you can specify it here and ``ploy`` will tell EC2 to assign it to the instance (if it's available) 37 | - ``6`` check `the EC2 pricing overview `_ for a description of the available instance types. 38 | 39 | - ``7`` every EC2 instance needs to belong to a so-called security group. You can either reference an existing one or create one like so:: 40 | 41 | [ec2-securitygroup:production] 42 | connections = 43 | tcp 22 22 0.0.0.0/0 44 | tcp 80 80 0.0.0.0/0 45 | tcp 443 443 0.0.0.0/0 46 | 47 | - ``8`` For FreeBSD the currently best option is to use Colin Percival's excellent `daemonology AMIs for FreeBSD `_. Simply pick the ID best suited for your hardware and region from the list and you're good to go! 48 | 49 | - ``9`` The default user for which daemonology's startup script configures SSH access (using the given ``keypair``) is named ``ec2-user`` but BSDploy's playbooks all assume ``root``, so we explicitly configure this here. Note, that this means that we *must* change the ``ec-user`` name (this happens in our own ``startup_script``, see below). 50 | 51 | - ``10`` ``ploy_ec2`` allows us to provide a local startup script which it will upload for us using Amazon's `instance metadata `_ mechanism. Here we reference it relative to the location of ``ploy.conf``. The following example provides minimal versions of ``rc.conf`` and ``sshd_config`` which... 52 | 53 | - configure SSH access for root 54 | - install Python (needed for running the :doc:`configuration playbook `) 55 | - updates FreeBSD to the latest patch level upon first boot:: 56 | 57 | #!/bin/sh 58 | cat << EOF > etc/rc.conf 59 | ec2_configinit_enable="YES" 60 | ec2_fetchkey_enable="YES" 61 | ec2_fetchkey_user="root" 62 | ec2_bootmail_enable="NO" 63 | ec2_ephemeralswap_enable="YES" 64 | ec2_loghostkey_enable="YES" 65 | ifconfig_xn0="DHCP" 66 | firstboot_freebsd_update_enable="YES" 67 | firstboot_pkgs_enable="YES" 68 | firstboot_pkgs_list="python27" 69 | dumpdev="AUTO" 70 | panicmail_enable="NO" 71 | panicmail_autosubmit="NO" 72 | sshd_enable="YES" 73 | EOF 74 | 75 | cat << EOF > /etc/ssh/sshd_config 76 | Port 22 77 | ListenAddress 0.0.0.0 78 | Subsystem sftp /usr/libexec/sftp-server 79 | PermitRootLogin without-password 80 | UseDNS no 81 | EOF 82 | 83 | Now you can provision the instance by running:: 84 | 85 | # ploy start production-backend 86 | 87 | This will take several minutes, as the machine is started up, updates itself and reboots. Be patient, it can easily take five minutes. To check if everything is done, use ploy's status command, once the instance is fully available it should say something like this:: 88 | 89 | # ploy status production-backend 90 | INFO: Instance 'production-backend' (i-xxxxx) available. 91 | INFO: Instance running. 92 | INFO: Instances DNS name ec2-xxx-xx-xx-xx.eu-west-1.compute.amazonaws.com 93 | INFO: Instances private DNS name ip-xxx-xx-xx-xx.eu-west-1.compute.internal 94 | INFO: Instances public DNS name ec2-xx-xx-xx-xx.eu-west-1.compute.amazonaws.com 95 | INFO: Console output available. SSH fingerprint verification possible. 96 | 97 | Especially the last line means that the new instance is now ready. 98 | 99 | You should now be able to log in via SSH:: 100 | 101 | ploy ssh production-backend 102 | 103 | .. Note:: Unlike with :doc:`plain ` or :doc:`Virtualbox ` instances, daemonology's `configinit `_ in conjunction with a ``startup_script`` such as the example above already perform everything we need in order to be able to run the jailhost playbooks. In other words, you can skip the :doc:`/setup/bootstrapping` step and continue straight to :doc:`/setup/configuration`. 104 | 105 | But before continuing on to :doc:`/setup/configuration`, let's take a look around while we're still logged in and note what hard disks and network interfaces are available. I.e. on our example machine of ``c3.xlarge`` type, the interface is named ``xn0`` and we have two SSDs of 40Gb at ``/dev/xbd1`` and ``/dev/xbd2``, but by default daemonology has already created a swap partition on the first slice (highly recommended, as most instance types don't have that much RAM), so we need to specify the second slice for our use. 106 | 107 | This means, that to configure a jailhost on this EC2 instance we need to declare an ``ez-master`` entry in ``ploy.conf`` with the following values:: 108 | 109 | [ez-master:production] 110 | instance = production-backend 111 | bootstrap_data_pool_devices = xbd1s2 xbd2s2 112 | 113 | In addition, since daemonology will also update the installation to the latest patch level, we will need to explicitly tell ``ezjail`` which version to install, since by default it uses the output of ``uname`` to compute the URL for downloading the base jail but that most likely won't exist (i.e ``10.0-RELEASE-p10``). You can do this by specifying ``ezjail_install_release`` for the ``ez-master`` like so:: 114 | 115 | ezjail_install_release = 10.0-RELEASE 116 | 117 | With this information you are now finally and truly ready to :doc:`configure the jailhost. `. -------------------------------------------------------------------------------- /bsdploy/__init__.py: -------------------------------------------------------------------------------- 1 | from glob import glob 2 | from os import path 3 | import argparse 4 | import logging 5 | import sys 6 | 7 | 8 | log = logging.getLogger("bsdploy") 9 | 10 | 11 | # register our own library and roles paths into ansible 12 | bsdploy_path = path.abspath(path.dirname(__file__)) 13 | 14 | ansible_paths = dict( 15 | roles=[path.join(bsdploy_path, 'roles')], 16 | library=[path.join(bsdploy_path, 'library')]) 17 | 18 | virtualbox_instance_defaults = { 19 | 'vm-ostype': 'FreeBSD_64', 20 | 'vm-memory': '2048', 21 | 'vm-accelerate3d': 'off', 22 | 'vm-acpi': 'on', 23 | 'vm-rtcuseutc': 'on', 24 | 'vm-boot1': 'disk', 25 | 'vm-boot2': 'dvd', 26 | 'vm-nic1': 'hostonly', 27 | 'vm-hostonlyadapter1': 'vboxnet0', 28 | } 29 | 30 | virtualbox_hostonlyif_defaults = { 31 | 'ip': '192.168.56.1', 32 | } 33 | 34 | virtualbox_dhcpserver_defaults = { 35 | 'ip': '192.168.56.2', 36 | 'netmask': '255.255.255.0', 37 | 'lowerip': '192.168.56.100', 38 | 'upperip': '192.168.56.254', 39 | } 40 | 41 | virtualbox_bootdisk_defaults = { 42 | 'size': '102400', 43 | } 44 | 45 | 46 | ez_instance_defaults = { 47 | 'ansible_python_interpreter': '/usr/local/bin/python2.7', 48 | 'fabric-shell': '/bin/sh -c', 49 | } 50 | 51 | 52 | class PloyBootstrapCmd(object): 53 | def __init__(self, ctrl): 54 | self.ctrl = ctrl 55 | 56 | def __call__(self, argv, help): 57 | """Bootstrap a jailhost that's been booted into MFSBsd.""" 58 | parser = argparse.ArgumentParser( 59 | prog="%s bootstrap" % self.ctrl.progname, 60 | description=help) 61 | masters = dict((master.id, master) for master in self.ctrl.get_masters('ezjail_admin')) 62 | parser.add_argument( 63 | "master", 64 | nargs='?' if len(masters) == 1 else 1, 65 | metavar="master", 66 | help="Name of the jailhost from the config.", 67 | choices=masters, 68 | default=list(masters.keys())[0] if len(masters) == 1 else None) 69 | parser.add_argument( 70 | "-y", "--yes", action="store_true", 71 | help="Answer yes to all questions.") 72 | parser.add_argument( 73 | "-p", "--http-proxy", 74 | help="Use http proxy for bootstrapping and pkg installation") 75 | args = parser.parse_args(argv) 76 | master = args.master if len(masters) == 1 else args.master[0] 77 | instance = self.ctrl.instances[master] 78 | instance.config.setdefault('ssh-timeout', 90) 79 | instance.hooks.before_bsdploy_bootstrap(instance) 80 | bootstrap_args = {'bootstrap-yes': args.yes} 81 | if args.http_proxy: 82 | bootstrap_args['http_proxy'] = args.http_proxy 83 | instance.do('bootstrap', **bootstrap_args) 84 | instance.hooks.after_bsdploy_bootstrap(instance) 85 | 86 | 87 | def get_bootstrap_path(instance): 88 | from ploy_ansible import get_playbooks_directory 89 | host_defined_path = instance.config.get('bootstrap-files') 90 | main_config = instance.master.main_config 91 | ploy_conf_path = main_config.path 92 | if host_defined_path is None: 93 | playbooks_directory = get_playbooks_directory(main_config) 94 | bootstrap_path = path.join(playbooks_directory, instance.uid, 'bootstrap-files') 95 | else: 96 | bootstrap_path = path.join(ploy_conf_path, host_defined_path) 97 | return bootstrap_path 98 | 99 | 100 | def get_ssh_key_paths(instance): 101 | bootstrap_path = get_bootstrap_path(instance) 102 | glob_path = path.join(bootstrap_path, 'ssh_host*_key.pub') 103 | key_paths = [] 104 | for ssh_key in glob(glob_path): 105 | ssh_key = path.abspath(ssh_key) 106 | key_paths.append(ssh_key) 107 | return key_paths 108 | 109 | 110 | def augment_instance(instance): 111 | from ploy_ansible import get_playbooks_directory 112 | from ploy_ansible import has_playbook 113 | from ploy.config import ConfigSection 114 | 115 | main_config = instance.master.main_config 116 | 117 | # provide virtualbox specific convenience defaults: 118 | if instance.master.sectiongroupname == ('vb-instance'): 119 | 120 | # default values for virtualbox instance 121 | for key, value in virtualbox_instance_defaults.items(): 122 | instance.config.setdefault(key, value) 123 | 124 | # default hostonly interface 125 | hostonlyif = main_config.setdefault('vb-hostonlyif', ConfigSection()) 126 | vboxnet0 = hostonlyif.setdefault('vboxnet0', ConfigSection()) 127 | for key, value in virtualbox_hostonlyif_defaults.items(): 128 | vboxnet0.setdefault(key, value) 129 | 130 | # default dhcp server 131 | dhcpserver = main_config.setdefault('vb-dhcpserver', ConfigSection()) 132 | vboxnet0 = dhcpserver.setdefault('vboxnet0', ConfigSection()) 133 | for key, value in virtualbox_dhcpserver_defaults.items(): 134 | vboxnet0.setdefault(key, value) 135 | 136 | # default virtual disk 137 | if 'vb-disk:defaultdisk' in instance.config.get('storage', {}): 138 | disks = main_config.setdefault('vb-disk', ConfigSection()) 139 | defaultdisk = disks.setdefault('defaultdisk', ConfigSection()) 140 | for key, value in virtualbox_bootdisk_defaults.items(): 141 | defaultdisk.setdefault(key, value) 142 | 143 | if not instance.master.sectiongroupname.startswith('ez-'): 144 | return 145 | 146 | for key, value in ez_instance_defaults.items(): 147 | instance.config.setdefault(key, value) 148 | 149 | if 'fabfile' not in instance.config: 150 | playbooks_directory = get_playbooks_directory(main_config) 151 | fabfile = path.join(playbooks_directory, instance.uid, 'fabfile.py') 152 | if path.exists(fabfile): 153 | instance.config['fabfile'] = fabfile 154 | else: 155 | fabfile = path.join(playbooks_directory, instance.id, 'fabfile.py') 156 | if path.exists(fabfile): 157 | instance.config['fabfile'] = fabfile 158 | 159 | if instance.master.instance is instance: 160 | # for hosts 161 | if 'fabfile' not in instance.config: 162 | bootstrap_type = instance.config.get('bootstrap', 'mfsbsd') 163 | fabfile = path.join(bsdploy_path, 'fabfile_%s.py' % bootstrap_type) 164 | instance.config['fabfile'] = fabfile 165 | if not path.exists(instance.config['fabfile']): 166 | log.error("The fabfile '%s' for instance '%s' doesn't exist." % ( 167 | instance.config['fabfile'], instance.uid)) 168 | sys.exit(1) 169 | if not has_playbook(instance): 170 | instance.config['roles'] = 'jails_host' 171 | if 'ssh-host-keys' not in instance.config: 172 | key_paths = get_ssh_key_paths(instance) 173 | instance.config['ssh-host-keys'] = "\n".join(key_paths) 174 | else: 175 | # for jails 176 | instance.config.setdefault('startup_script', path.join( 177 | bsdploy_path, 'startup-ansible-jail.sh')) 178 | instance.config.setdefault('flavour', 'bsdploy_base') 179 | 180 | 181 | def get_commands(ctrl): 182 | return [('bootstrap', PloyBootstrapCmd(ctrl))] 183 | 184 | 185 | plugin = dict( 186 | augment_instance=augment_instance, 187 | get_commands=get_commands) 188 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = _build 9 | 10 | # User-friendly check for sphinx-build 11 | ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) 12 | $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) 13 | endif 14 | 15 | # Internal variables. 16 | PAPEROPT_a4 = -D latex_paper_size=a4 17 | PAPEROPT_letter = -D latex_paper_size=letter 18 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 19 | # the i18n builder cannot share the environment and doctrees with the others 20 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 21 | 22 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext 23 | 24 | help: 25 | @echo "Please use \`make ' where is one of" 26 | @echo " html to make standalone HTML files" 27 | @echo " dirhtml to make HTML files named index.html in directories" 28 | @echo " singlehtml to make a single large HTML file" 29 | @echo " pickle to make pickle files" 30 | @echo " json to make JSON files" 31 | @echo " htmlhelp to make HTML files and a HTML help project" 32 | @echo " qthelp to make HTML files and a qthelp project" 33 | @echo " devhelp to make HTML files and a Devhelp project" 34 | @echo " epub to make an epub" 35 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 36 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 37 | @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" 38 | @echo " text to make text files" 39 | @echo " man to make manual pages" 40 | @echo " texinfo to make Texinfo files" 41 | @echo " info to make Texinfo files and run them through makeinfo" 42 | @echo " gettext to make PO message catalogs" 43 | @echo " changes to make an overview of all changed/added/deprecated items" 44 | @echo " xml to make Docutils-native XML files" 45 | @echo " pseudoxml to make pseudoxml-XML files for display purposes" 46 | @echo " linkcheck to check all external links for integrity" 47 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 48 | 49 | clean: 50 | rm -rf $(BUILDDIR)/* 51 | 52 | html: 53 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 54 | @echo 55 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 56 | 57 | dirhtml: 58 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 59 | @echo 60 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 61 | 62 | singlehtml: 63 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 64 | @echo 65 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 66 | 67 | pickle: 68 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 69 | @echo 70 | @echo "Build finished; now you can process the pickle files." 71 | 72 | json: 73 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 74 | @echo 75 | @echo "Build finished; now you can process the JSON files." 76 | 77 | htmlhelp: 78 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 79 | @echo 80 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 81 | ".hhp project file in $(BUILDDIR)/htmlhelp." 82 | 83 | qthelp: 84 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 85 | @echo 86 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 87 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 88 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/bsdploy.qhcp" 89 | @echo "To view the help file:" 90 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/bsdploy.qhc" 91 | 92 | devhelp: 93 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 94 | @echo 95 | @echo "Build finished." 96 | @echo "To view the help file:" 97 | @echo "# mkdir -p $$HOME/.local/share/devhelp/bsdploy" 98 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/bsdploy" 99 | @echo "# devhelp" 100 | 101 | epub: 102 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 103 | @echo 104 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 105 | 106 | latex: 107 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 108 | @echo 109 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 110 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 111 | "(use \`make latexpdf' here to do that automatically)." 112 | 113 | latexpdf: 114 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 115 | @echo "Running LaTeX files through pdflatex..." 116 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 117 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 118 | 119 | latexpdfja: 120 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 121 | @echo "Running LaTeX files through platex and dvipdfmx..." 122 | $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja 123 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 124 | 125 | text: 126 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 127 | @echo 128 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 129 | 130 | man: 131 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 132 | @echo 133 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 134 | 135 | texinfo: 136 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 137 | @echo 138 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 139 | @echo "Run \`make' in that directory to run these through makeinfo" \ 140 | "(use \`make info' here to do that automatically)." 141 | 142 | info: 143 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 144 | @echo "Running Texinfo files through makeinfo..." 145 | make -C $(BUILDDIR)/texinfo info 146 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 147 | 148 | gettext: 149 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 150 | @echo 151 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 152 | 153 | changes: 154 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 155 | @echo 156 | @echo "The overview file is in $(BUILDDIR)/changes." 157 | 158 | linkcheck: 159 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 160 | @echo 161 | @echo "Link check complete; look for any errors in the above output " \ 162 | "or in $(BUILDDIR)/linkcheck/output.txt." 163 | 164 | doctest: 165 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 166 | @echo "Testing of doctests in the sources finished, look at the " \ 167 | "results in $(BUILDDIR)/doctest/output.txt." 168 | 169 | xml: 170 | $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml 171 | @echo 172 | @echo "Build finished. The XML files are in $(BUILDDIR)/xml." 173 | 174 | pseudoxml: 175 | $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml 176 | @echo 177 | @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." 178 | -------------------------------------------------------------------------------- /docs/tutorial/webserver.rst: -------------------------------------------------------------------------------- 1 | Webserver 2 | ========= 3 | 4 | The idea is that the webserver lives in its own jail and receives all HTTP traffic and then redirects the requests as required to the individual jails. 5 | 6 | The webserver jail itself will host a simple website that lists and links the available services. 7 | 8 | The website will be in a dedicated ZFS that will be mounted into the jail, so let's start with creating this. 9 | 10 | In ``etc/ploy.conf`` from the quickstart you should have the following ezjail master:: 11 | 12 | [ez-master:jailhost] 13 | instance = ploy-demo 14 | roles = 15 | dhcp_host 16 | jails_host 17 | 18 | 19 | ZFS mounting 20 | ------------ 21 | 22 | To set up the ZFS layout we will replace the inline roles with a dedicated playbook, so let's delete the ``roles =`` entry and create a top-level file names ``jailhost.yml`` with the following contents:: 23 | 24 | --- 25 | - hosts: jailhost 26 | user: root 27 | roles: 28 | - dhcp_host 29 | - jails_host 30 | 31 | Once we have a playbook in place, it becomes easy to add custom tasks:: 32 | 33 | tasks: 34 | - name: ensure ZFS file systems are in place 35 | zfs: name={{ item }} state=present mountpoint=/{{ item }} 36 | with_items: 37 | - tank/htdocs 38 | tags: zfs-layout 39 | 40 | To apply the playbook it's easiest to call the ``configure`` command again. 41 | While Ansible is smart enough to only apply those parts that actually need to be applied again that process can become quite slow, as playbooks grow in size. 42 | So here, we tag the new task with ``zfs-layout``, so we can call it explicitly:: 43 | 44 | ploy configure jailhost -t zfs-layout 45 | 46 | .. note:: You should see an ``INFO`` entry in the output that tells you which playbook the ``configure`` command has used. 47 | 48 | 49 | Exercise 50 | -------- 51 | 52 | In the quickstart, we've created a demo jail which conveniently already contains a webserver. 53 | 54 | Repurpose it to create a jail named ``webserver`` which has nginx installed into it. 55 | 56 | Do this by first terminating the demo jail, *then* renaming it, otherwise we will have an 'orphaned' instance which would interfere with the new jail:: 57 | 58 | ploy terminate demo_jail 59 | 60 | Now we can mount the website ZFS into it like so:: 61 | 62 | [ez-instance:webserver] 63 | master = jailhost 64 | ip = 10.0.0.2 65 | mounts = 66 | src=/tank/htdocs dst=/usr/local/www/data ro=true 67 | 68 | .. note:: Mounting filesystems into jails gives us the ability to mount them read-only, like we do in this case. 69 | 70 | Let's start the new jail:: 71 | 72 | ploy start webserver 73 | 74 | 75 | Instance playbooks 76 | ------------------ 77 | 78 | Note that the webserver jail doesn't have a role assigned. 79 | Roles are useful for more complex scenarios that are being re-used. 80 | For smaller tasks it's often easier to simply create a one-off playbook for a particular host. 81 | 82 | To associate a playbook with an instance you need to create a top-level YAML file with the same name as the instance – like we did above for the jailhost. 83 | For jail instances, this name contains both the name of the master instance *and* the name of the jail, so let's create a top-level file named ``jailhost-webserver.yml`` with the following contents:: 84 | 85 | --- 86 | - hosts: jailhost-webserver 87 | tasks: 88 | - name: install nginx 89 | pkgng: 90 | name: "nginx" 91 | state: "present" 92 | - name: enable nginx at startup time 93 | lineinfile: dest=/etc/rc.conf state=present line='nginx_enable=YES' create=yes 94 | - name: make sure nginx is running or reloaded 95 | service: name=nginx state=restarted 96 | 97 | In the above playbook we demonstrate 98 | 99 | - how to install a package with the ``pkgng`` module from ansible 100 | - enable a service in ``rc.conf`` using the ``lineinfile`` command 101 | - ensure a service is running with the ``service`` command 102 | 103 | Let's give that a spin:: 104 | 105 | ploy configure webserver 106 | 107 | .. note:: If the name of a jail is only used in a single master instance, ``ploy`` allows us to address it without stating the full name on the command line for convenience. IOW the above command is an alias to ``ploy configure jailhost-webserver``. 108 | 109 | 110 | "Publishing" jails 111 | ------------------ 112 | 113 | Eventhough the webserver is now running, we cannot reach it from the outside, we first need to explicitly enable access. While there are several possibilites to achieve this, we will use ``ipnat``, just like in the quickstart. 114 | 115 | So, create or edit ``host_vars/jailhost.yml`` to look like so:: 116 | 117 | pf_nat_rules: 118 | - "rdr on {{ ansible_default_ipv4.interface }} proto tcp from any to {{ ansible_default_ipv4.interface }} port 80 -> {{ hostvars['jailhost-webserver']['ploy_ip'] }} port 80" 119 | 120 | To activate the rules, re-apply the jail host configuration for just the ``pf-conf`` tag:: 121 | 122 | ploy configure jailhost -t pf-conf 123 | 124 | You should now be able to access the default nginx website at the ``http://192.168.56.100`` address. 125 | 126 | 127 | Use defaults 128 | ------------ 129 | 130 | Currently the webserver serves the default site located at ``/usr/local/www/nginx`` which is a symbolic link to ``nginx-dist``. 131 | 132 | Now, to switch it the website located inside the ZFS filesystem we could either change the nginx configuration to point to it but in practice it can be a good idea to use default settings as much as possible and instead make the environment match the default. 133 | *Every custom configuration file you can avoid is a potential win*. 134 | 135 | In this particular case, let's mount the website into the default location. First we need to remove the symbolic link that has been created by the nginx start up. 136 | Since this is truly a one-time operation (if we re-run the modified playbook against a fresh instance the symbolic link would not be created and wouldn't need to be removed) we can use ploy's ability to execute ssh commands like so:: 137 | 138 | ploy ssh webserver "rm /usr/local/www/nginx" 139 | 140 | Now we can change the mountpoint in ``ploy.conf``:: 141 | 142 | [ez-instance:webserver] 143 | master = jailhost 144 | ip = 10.0.0.2 145 | mounts = 146 | src=/tank/htdocs dst=/usr/local/www/nginx ro=true 147 | 148 | Unfortunately, currently the only way to re-mount is to stop and start the jail in question, so let's do that:: 149 | 150 | ploy stop webserver 151 | ploy start webserver 152 | 153 | Reload the website in your browser: you should now receive a ``Forbidden`` instead of the default site because the website directory is still empty. 154 | 155 | 156 | Fabric integration 157 | ------------------ 158 | 159 | So far we've used ansible to configure the host and the jail. 160 | Its declarative approach is perfect for this. 161 | But what about maintenance tasks such as updating the contents of a website? 162 | Such tasks are a more natural fit for an *imperative* approach and ``ploy_fabric`` gives us a neat way of doing this. 163 | 164 | Let's create a top-level file named ``fabfile.py`` with the following contents:: 165 | 166 | from fabric import api as fab 167 | 168 | def upload_website(): 169 | fab.put('htdocs/*', '/usr/jails/webserver/usr/local/www/nginx/') 170 | 171 | Since the webserver jail only has read-access, we need to upload the website via the host (for now), so let's associate the fabric file with the host by making its entry in ``ploy.conf`` look like so:: 172 | 173 | [ez-master:jailhost] 174 | instance = ploy-demo 175 | fabfile = ../fabfile.py 176 | 177 | Create a simple index page:: 178 | 179 | mkdir htdocs 180 | echo "Hello Berlin" >> htdocs/index.html 181 | 182 | Then upload it:: 183 | 184 | ploy do jailhost upload_website 185 | 186 | and reload the website. 187 | 188 | 189 | Exercise One 190 | ------------ 191 | 192 | Requiring write-access to the jail host in order to update the website is surely not very clever. 193 | 194 | Your task is to create a jail named ``website_edit`` that contains a writeable mount of the website and which uses a modified version of the fabric script from above to update the contents. 195 | 196 | 197 | Exercise Two 198 | ------------ 199 | 200 | Put the path to the website on the host into a ansible variable defined in ploy.conf and make the fabric script reference it instead of hard coding it. 201 | 202 | You can access variables defined in ansible and ``ploy.conf`` in Fabric via its ``env`` like so:: 203 | 204 | ansible_vars = fab.env.instance.get_ansible_variables() 205 | 206 | The result is a dictionary populated with variables from ``group_vars``, ``host_vars`` and from within ``ploy.conf``. 207 | However, it does *not* contain any of the Ansible facts. 208 | For details check `ploy_fabric's documentation `_ 209 | -------------------------------------------------------------------------------- /bsdploy/library/zpool: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | DOCUMENTATION = ''' 3 | --- 4 | module: zpool 5 | author: Florian Schulze 6 | short_description: Manage ZFS pools 7 | description: 8 | - Manage ZFS pools 9 | ''' 10 | 11 | import os 12 | 13 | 14 | class Zpool(object): 15 | 16 | platform = 'FreeBSD' 17 | 18 | def __init__(self, module): 19 | self.module = module 20 | self.changed = False 21 | self.state = self.module.params.pop('state') 22 | self.name = self.module.params.pop('name') 23 | # set to empty string to create latest supported version on host 24 | self.version = self.module.params.pop('version') 25 | self.devices = self.module.params.pop('devices') 26 | self.raid_mode = self.module.params.pop('raid_mode') 27 | self.geli = self.module.params.pop('geli') 28 | self.geli_passphrase_location = self.module.params.pop('geli_passphrase_location') 29 | self.cmd = self.module.get_bin_path('zpool', required=True) 30 | 31 | def zpool(self, *params): 32 | return self.module.run_command([self.cmd] + list(params)) 33 | 34 | def exists(self): 35 | (rc, out, err) = self.zpool('list', '-H', self.name) 36 | if self.geli and 'FAULTED' in out: 37 | return False 38 | return rc == 0 39 | 40 | def get_devices(self): 41 | devices = self.devices 42 | if not devices: 43 | if not os.path.exists('/dev/gpt'): 44 | return [] 45 | return [ 46 | 'gpt/%s' % x 47 | for x in os.listdir('/dev/gpt') 48 | if x.startswith("%s_" % self.name) and not x.endswith('.eli')] 49 | return devices.split() 50 | 51 | def get_raid_mode(self, devices): 52 | raid_mode = self.raid_mode 53 | if raid_mode == 'detect': 54 | if len(devices) == 1: 55 | raid_mode = '' 56 | elif len(devices) == 2: 57 | raid_mode = 'mirror' 58 | else: 59 | self.module.fail_json(msg="Don't know how to handle %s number of devices (%s)." % (len(devices), ' '.join(devices))) 60 | return raid_mode 61 | 62 | def prepare_geli_passphrase(self): 63 | if os.path.exists(self.geli_passphrase_location): 64 | return 65 | (rc, out, err) = self.module.run_command([ 66 | 'openssl', 'rand', '-base64', '32']) 67 | if rc != 0: 68 | self.module.fail_json(msg="Failed to generate '%s'.\n%s" % (self.geli_passphrase_location, err)) 69 | with open(self.geli_passphrase_location, 'w') as f: 70 | f.write(out) 71 | os.chmod(self.geli_passphrase_location, 0o600) 72 | 73 | def prepare_data_devices(self, devices): 74 | data_devices = [] 75 | if self.geli: 76 | self.prepare_geli_passphrase() 77 | rc_devices = [] 78 | for device in devices: 79 | data_device = '{device}.eli'.format(device=device) 80 | data_devices.append(data_device) 81 | rc_devices.append(device) 82 | if os.path.exists(os.path.join('/dev', data_device)): 83 | continue 84 | # try to attach geli device in case it's already existing 85 | (rc, out, err) = self.module.run_command([ 86 | 'geli', 'attach', 87 | '-j', self.geli_passphrase_location, 88 | device]) 89 | if rc == 0: 90 | continue 91 | # create new geli device 92 | (rc, out, err) = self.module.run_command([ 93 | 'geli', 'init', '-s', '4096', '-l', '256', 94 | '-J', self.geli_passphrase_location, 95 | device]) 96 | self.changed = True 97 | if rc != 0: 98 | self.module.fail_json(msg="Failed to init geli device '%s'.\n%s" % (data_device, err)) 99 | (rc, out, err) = self.module.run_command([ 100 | 'geli', 'attach', 101 | '-j', self.geli_passphrase_location, 102 | device]) 103 | if rc != 0: 104 | self.module.fail_json(msg="Failed to attach geli device '%s'.\n%s" % (data_device, err)) 105 | (rc, out, err) = self.module.run_command([ 106 | 'sysrc', 'geli_devices']) 107 | if rc != 0: 108 | geli_devices = [] 109 | else: 110 | geli_devices = out[14:].split() 111 | geli_devices.extend(rc_devices) 112 | (rc, out, err) = self.module.run_command([ 113 | 'sysrc', 'geli_devices=%s' % ' '.join(sorted(set(geli_devices)))]) 114 | if rc != 0: 115 | self.module.fail_json(msg="Failed to set geli_devices in rc.conf to '%s'.\n%s" % (' '.join(geli_devices), err)) 116 | for rc_device in rc_devices: 117 | key = 'geli_%s_flags' % rc_device.replace('/', '_') 118 | value = '-j %s' % self.geli_passphrase_location 119 | (rc, out, err) = self.module.run_command([ 120 | 'sysrc', '%s=%s' % (key, value)]) 121 | if rc != 0: 122 | self.module.fail_json(msg="Failed to set %s in rc.conf to '%s'.\n%s" % (key, value, err)) 123 | else: 124 | for device in devices: 125 | data_device = '{device}.nop'.format(device=device) 126 | (rc, out, err) = self.module.run_command([ 127 | 'gnop', 'create', '-S', '4096', device]) 128 | if rc != 0: 129 | self.module.fail_json(msg="Failed to create nop device '%s'.\n%s" % (data_device, err)) 130 | data_devices.append(data_device) 131 | return data_devices 132 | 133 | def cleanup_data_devices(self, data_devices): 134 | if self.geli: 135 | return 136 | (rc, out, err) = self.zpool('export', self.name) 137 | if rc != 0: 138 | self.module.fail_json(msg="Failed to export zpool '%s'.\n%s" % (self.name, err)) 139 | for data_device in data_devices: 140 | (rc, out, err) = self.module.run_command([ 141 | 'gnop', 'destroy', data_device]) 142 | if rc != 0: 143 | self.module.fail_json(msg="Failed to destroy nop device '%s'.\n%s" % (data_device, err)) 144 | (rc, out, err) = self.zpool('import', self.name) 145 | if rc != 0: 146 | self.module.fail_json(msg="Failed to import zpool '%s'.\n%s" % (self.name, err)) 147 | 148 | def create(self): 149 | result = dict() 150 | if self.module.check_mode: 151 | self.changed = True 152 | return result 153 | devices = self.get_devices() 154 | raid_mode = self.get_raid_mode(devices) 155 | data_devices = self.prepare_data_devices(devices) 156 | # check again in case the zpool wasn't active because of missing geli settings 157 | if not self.exists(): 158 | (rc, out, err) = self.zpool('import', '-f', self.name) 159 | if rc != 0: 160 | zpool_args = ['create'] 161 | if self.version: 162 | zpool_args.extend(['-o', 'version=%s' % self.version]) 163 | zpool_args.extend(['-m', 'none', self.name]) 164 | if raid_mode: 165 | zpool_args.append(raid_mode) 166 | zpool_args.extend(data_devices) 167 | (rc, out, err) = self.zpool(*zpool_args) 168 | if rc != 0: 169 | self.module.fail_json(msg="Failed to create zpool with the following arguments:\n%s\n%s" % (' '.join(zpool_args), err)) 170 | self.changed = True 171 | self.cleanup_data_devices(data_devices) 172 | return result 173 | 174 | def destroy(self): 175 | raise NotImplementedError 176 | 177 | def __call__(self): 178 | result = dict(name=self.name, state=self.state) 179 | 180 | if self.state in ['present', 'running']: 181 | if not self.exists(): 182 | result.update(self.create()) 183 | elif self.state == 'absent': 184 | if self.exists(): 185 | self.destroy() 186 | 187 | result['changed'] = self.changed 188 | return result 189 | 190 | 191 | MODULE_SPECS = dict( 192 | argument_spec=dict( 193 | name=dict(required=True, type='str'), 194 | state=dict(default='present', choices=['present', 'absent'], type='str'), 195 | version=dict(default='28', type='str'), 196 | devices=dict(default=None, type='str'), 197 | raid_mode=dict(default='detect', type='str'), 198 | geli=dict(default=False, type='bool'), 199 | geli_passphrase_location=dict(default='/root/geli-passphrase', type='str')), 200 | supports_check_mode=True) 201 | 202 | 203 | def main(): 204 | module = AnsibleModule(**MODULE_SPECS) 205 | result = Zpool(module)() 206 | if 'failed' in result: 207 | module.fail_json(**result) 208 | else: 209 | module.exit_json(**result) 210 | 211 | 212 | from ansible.module_utils.basic import * # noqa 213 | if __name__ == "__main__": 214 | main() 215 | -------------------------------------------------------------------------------- /docs/setup/bootstrapping.rst: -------------------------------------------------------------------------------- 1 | Bootstrapping 2 | ============= 3 | 4 | Bootstrapping in the context of BSDploy means installing FreeBSD onto a :doc:`previously provisioned host` and the smallest amount of configuration to make it ready for its final configuration. 5 | 6 | The Bootstrapping process assumes that the target host has been booted into an installer and can be reached via SSH under the configured address and that you have configured the appropriate bootstrapping type (currently either ``mfsbsd`` or ``daemonology``). 7 | 8 | 9 | Bootstrapping FreeBSD 9.x 10 | ------------------------- 11 | 12 | The default version that BSDploy assumes is 10.3. 13 | If you want to install different versions, i.e. 9.2 you must: 14 | 15 | - use the iso image for that version:: 16 | 17 | % ploy-download http://mfsbsd.vx.sk/files/iso/9/amd64/mfsbsd-se-9.2-RELEASE-amd64.iso 4ef70dfd7b5255e36f2f7e1a5292c7a05019c8ce downloads/ 18 | 19 | - set ``bootstrap-ssh-host-key`` in ``ploy.conf`` to content of ``/etc/ssh/ssh_host_rsa_key.pub`` in the mfsbsd image 20 | (each mfsbsd release has it's own hardcoded ssh host key) 21 | - create a file named ``files.yml`` in ``bootstrap-files`` with the following contents: 22 | 23 | .. code-block:: yaml 24 | 25 | --- 26 | 'pkg.txz': 27 | url: 'http://pkg.freebsd.org/freebsd:9:x86:64/quarterly/Latest/pkg.txz' 28 | directory: '/mnt/var/cache/pkg/All' 29 | remote: '/mnt/var/cache/pkg/All/pkg.txz' 30 | 31 | 32 | Bootstrap configuration 33 | ----------------------- 34 | 35 | Since bootstrapping is specific to BSDploy we cannot configure it in the provisioning instance. Instead we need to create a specific entry for it in our configuration of the type ``ez-master`` and assign it to the provisioner. 36 | 37 | I.e. in our example: 38 | 39 | .. code-block:: ini 40 | 41 | [ez-master:jailhost] 42 | instance = ploy-demo 43 | 44 | 45 | Required parameters 46 | +++++++++++++++++++ 47 | 48 | The only other required value for an ``ez-master`` besides its provisioner is the name of the target device(s) the system should be installed on. 49 | 50 | If you don't know the device name FreeBSD has assigned, run ``gpart list`` and look for in the ``Consumers`` section towards the end of the output. 51 | 52 | If you provide more than one device name, BSDploy will create a zpool mirror configuration, just make sure the devices are compatible. 53 | 54 | There we can provide the name of the target device, so we get the following: 55 | 56 | .. code-block:: ini 57 | 58 | [ez-master:jailhost] 59 | instance = ploy-demo 60 | bootstrap-system-devices = ada0 61 | 62 | Or if we have more than one device: 63 | 64 | .. code-block:: ini 65 | 66 | [ez-master:jailhost] 67 | instance = ploy-demo 68 | bootstrap-system-devices = 69 | ada0 70 | ada1 71 | 72 | 73 | Optional parameters 74 | +++++++++++++++++++ 75 | 76 | You can use the following optional parameters to configure the bootstrapping process and thus influence what the jail host will look like: 77 | 78 | - ``bootstrap-system-pool-size``: BSDploy will create a zpool on the target device named ``system`` with the given size. This value will be passed on to ``zfsinstall``, so you can provide standard units, such as ``5G``. Default is ``20G``. 79 | 80 | - ``bootstrap-swap-size``: This is the size of the swap space that will be created. Default is double the amount of detected RAM. 81 | 82 | - ``bootstrap-bsd-url``: If you don't want to use the installation files found on the installer image (or if your boot image doesn't contain any) you can provide an explicit alternative (i.e. ``http://ftp4.de.freebsd.org/pub/FreeBSD/releases/amd64/9.2-RELEASE/``) and this will be used to fetch the system from. 83 | 84 | - ``bootstrap-ssh-host-key``: Since the installer runs a different sshd configuration than the final installation, we need to provide its ssh host key explicitly. However, if you don't provide one, BSDploy will assume the (currently hardcoded) host key of the 10.3 mfsBSD installer . If you are using newer versions you must update the value to the content of ``/etc/ssh/ssh_host_rsa_key.pub`` in the mfsbsd image. 85 | 86 | - ``bootstrap-ssh-fingerprints``: Another way to supply SSH fingerprints if host keys aren't an option. 87 | 88 | - ``bootstrap-firstboot-update``: If set install and enable the `firstboot-freebsd-update `_ package. This will update the installed system automatically (meaning non-interactively) to the latest patchlevel upon first boot. Disabled by default. 89 | 90 | - ``bootstrap-password``: If set will use the provided password for bootstrapping. It is used to update the known hosts. You might still have to provide it manually for the SSH command used during bootstrap. 91 | 92 | - ``http_proxy``: If set, that proxy will be used for all ``pkg`` operations performed on that host, as well as for downloading any assets during bootstrapping (``base.tbz`` etc.) 93 | 94 | 95 | .. note:: Regarding the http proxy setting it is noteworthy, that ``pkg`` servers have rather restrictive caching policies in their response headers, so that most proxies' default configurations will produce misses. Here's an example for how to configure squid to produce better results: 96 | 97 | .. code-block:: sh 98 | 99 | # match against download urls for specific packages - their content never changes for the same url, so we cache aggressively 100 | refresh_pattern -i (quarterly|latest)\/All\/.*(\.txz) 1440 100% 1051200 ignore-private ignore-must-revalidate override-expire ignore-no-cache 101 | # match against meta-information - this shouldn't be cached quite so aggressively 102 | refresh_pattern -i (quarterly|latest)\/.*(\.txz) 1440 100% 10080 ignore-private ignore-must-revalidate override-expire ignore-no-cache 103 | 104 | Also you will probably want to adjust the following: 105 | 106 | .. code-block:: sh 107 | 108 | maximum_object_size_in_memory 32 KB 109 | maximum_object_size 2000 MB 110 | 111 | 112 | Bootstrap rc.conf 113 | ----------------- 114 | 115 | A crucial component of bootstrapping is configuring ``/etc/rc.conf``. 116 | 117 | One option is to provide a custom rc.conf (verbatim or as a template) for your host via :ref:`bootstrap-files`. 118 | 119 | But often times, the default template with a few additional custom lines will suffice. 120 | 121 | Here's what the default ``rc.conf`` template looks like: 122 | 123 | .. literalinclude:: ../../bsdploy/bootstrap-files/rc.conf 124 | 125 | This is achieved by providing ``boostrap-rc-xxxx`` key/values in the instance definition in ``ploy.conf``. 126 | 127 | 128 | .. _bootstrap-files: 129 | 130 | Bootstrap files 131 | --------------- 132 | 133 | During bootstrapping a certain number of files are copied onto the target host. 134 | 135 | Some of these files... 136 | 137 | - need to be provided by the user (i.e. ``authorized_keys``) 138 | - others have some (sensible) defaults (i.e. ``rc.conf``) 139 | - some can be downloaded via URL (i.e.) ``http://pkg.freebsd.org/freebsd:10:x86:64/latest/Latest/pkg.txz`` 140 | 141 | The list of files, their possible sources and their destination is encoded in a ``.yml`` file, the default of which is this 142 | 143 | .. literalinclude:: ../../bsdploy/bootstrap-files/files.yml 144 | 145 | .. warning:: Overriding the list of default files is an advanced feature and in most cases it is not needed. Also keep in mind that bootstrapping is only about getting the host ready for running BSDploy. Any additional files beyond that should be uploaded lateron via fabric and/or playbooks. 146 | 147 | It is however, quite common and useful to customize files that are part of the above list with custom versions *per host*. 148 | 149 | For example, to create a custom ``rc.conf`` for a particular instance, create a ``bootstrap-files`` entry for it and point it to a directory in your project, usually ``../bootstrap-files/INSTANCENAME/`` and place your version of ``rc.conf`` inside there. Note, that by default this file is rendered as a template, your custom version will be, too. 150 | 151 | Any file listed in the YAML file found inside that directory will take precedence during bootstrapping, but any file *not* found in there will be uploaded from the default location instead. 152 | 153 | Files encrypted using ``ploy vault encrypt`` are recognized and decrypted during upload. 154 | 155 | SSH host keys are generated locally via ``ssh-keygen`` and stored in ``bootstrap-files``. 156 | If your ``ssh-keygen`` doesn't support a key type (like ecdsa on OS X), then the key won't be created and FreeBSD will create it during first boot. 157 | The generated keys are used to verify the ssh connection, so it is best to add them into version control. 158 | Since the private ssh keys are sensitive data, you should encrypt them using ``ploy vault encrypt ...`` before adding them into version control. 159 | 160 | 161 | Bootstrap execution 162 | ------------------- 163 | 164 | With (all) those pre-requisites out of the way, the entire process boils down to issuing the following command:: 165 | 166 | % ploy bootstrap 167 | 168 | Or, if your configuration has more than one instance defined you need to provide its name, i.e.:: 169 | 170 | % ploy bootstrap jailhost 171 | 172 | Once this has run successfully, you can move on to the final setup step :doc:`Configuration `. 173 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # bsdploy documentation build configuration file, created by 4 | # sphinx-quickstart on Mon Jun 9 17:48:25 2014. 5 | # 6 | # This file is execfile()d with the current directory set to its 7 | # containing dir. 8 | # 9 | # Note that not all possible configuration values are present in this 10 | # autogenerated file. 11 | # 12 | # All configuration values have a default; values that are commented out 13 | # serve to show the default. 14 | 15 | import sys 16 | import os 17 | 18 | # If extensions (or modules to document with autodoc) are in another directory, 19 | # add these directories to sys.path here. If the directory is relative to the 20 | # documentation root, use os.path.abspath to make it absolute, like shown here. 21 | #sys.path.insert(0, os.path.abspath('.')) 22 | 23 | # -- General configuration ------------------------------------------------ 24 | 25 | # If your documentation needs a minimal Sphinx version, state it here. 26 | #needs_sphinx = '1.0' 27 | 28 | # Add any Sphinx extension module names here, as strings. They can be 29 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 30 | # ones. 31 | extensions = [ 32 | 'sphinx.ext.autodoc', 33 | 'sphinx.ext.doctest', 34 | ] 35 | 36 | # Add any paths that contain templates here, relative to this directory. 37 | templates_path = ['_templates'] 38 | 39 | # The suffix of source filenames. 40 | source_suffix = '.rst' 41 | 42 | # The encoding of source files. 43 | #source_encoding = 'utf-8-sig' 44 | 45 | # The master toctree document. 46 | master_doc = 'index' 47 | 48 | # General information about the project. 49 | project = u'BSDploy' 50 | copyright = u'2014, Tom Lazar' 51 | 52 | # The version info for the project you're documenting, acts as replacement for 53 | # |version| and |release|, also used in various other places throughout the 54 | # built documents. 55 | # 56 | # The short X.Y version. 57 | version = '1.0' 58 | # The full version, including alpha/beta/rc tags. 59 | from pkg_resources import get_distribution 60 | release = get_distribution('bsdploy').version 61 | 62 | # The language for content autogenerated by Sphinx. Refer to documentation 63 | # for a list of supported languages. 64 | #language = None 65 | 66 | # There are two options for replacing |today|: either, you set today to some 67 | # non-false value, then it is used: 68 | #today = '' 69 | # Else, today_fmt is used as the format for a strftime call. 70 | #today_fmt = '%B %d, %Y' 71 | 72 | # List of patterns, relative to source directory, that match files and 73 | # directories to ignore when looking for source files. 74 | exclude_patterns = ['_build'] 75 | 76 | # The reST default role (used for this markup: `text`) to use for all 77 | # documents. 78 | #default_role = None 79 | 80 | # If true, '()' will be appended to :func: etc. cross-reference text. 81 | #add_function_parentheses = True 82 | 83 | # If true, the current module name will be prepended to all description 84 | # unit titles (such as .. function::). 85 | #add_module_names = True 86 | 87 | # If true, sectionauthor and moduleauthor directives will be shown in the 88 | # output. They are ignored by default. 89 | #show_authors = False 90 | 91 | # The name of the Pygments (syntax highlighting) style to use. 92 | pygments_style = 'sphinx' 93 | 94 | # A list of ignored prefixes for module index sorting. 95 | #modindex_common_prefix = [] 96 | 97 | # If true, keep warnings as "system message" paragraphs in the built documents. 98 | #keep_warnings = False 99 | 100 | 101 | # -- Options for HTML output ---------------------------------------------- 102 | 103 | # The theme to use for HTML and HTML Help pages. See the documentation for 104 | # a list of builtin themes. 105 | html_theme = 'default' 106 | 107 | # Theme options are theme-specific and customize the look and feel of a theme 108 | # further. For a list of options available for each theme, see the 109 | # documentation. 110 | #html_theme_options = {} 111 | 112 | # Add any paths that contain custom themes here, relative to this directory. 113 | #html_theme_path = [] 114 | 115 | # The name for this set of Sphinx documents. If None, it defaults to 116 | # " v documentation". 117 | #html_title = None 118 | 119 | # A shorter title for the navigation bar. Default is the same as html_title. 120 | #html_short_title = None 121 | 122 | # The name of an image file (relative to this directory) to place at the top 123 | # of the sidebar. 124 | #html_logo = None 125 | 126 | # The name of an image file (within the static path) to use as favicon of the 127 | # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 128 | # pixels large. 129 | #html_favicon = None 130 | 131 | # Add any paths that contain custom static files (such as style sheets) here, 132 | # relative to this directory. They are copied after the builtin static files, 133 | # so a file named "default.css" will overwrite the builtin "default.css". 134 | html_static_path = ['_static'] 135 | 136 | # Add any extra paths that contain custom files (such as robots.txt or 137 | # .htaccess) here, relative to this directory. These files are copied 138 | # directly to the root of the documentation. 139 | #html_extra_path = [] 140 | 141 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, 142 | # using the given strftime format. 143 | #html_last_updated_fmt = '%b %d, %Y' 144 | 145 | # If true, SmartyPants will be used to convert quotes and dashes to 146 | # typographically correct entities. 147 | #html_use_smartypants = True 148 | 149 | # Custom sidebar templates, maps document names to template names. 150 | #html_sidebars = {} 151 | 152 | # Additional templates that should be rendered to pages, maps page names to 153 | # template names. 154 | #html_additional_pages = {} 155 | 156 | # If false, no module index is generated. 157 | #html_domain_indices = True 158 | 159 | # If false, no index is generated. 160 | #html_use_index = True 161 | 162 | # If true, the index is split into individual pages for each letter. 163 | #html_split_index = False 164 | 165 | # If true, links to the reST sources are added to the pages. 166 | #html_show_sourcelink = True 167 | 168 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. 169 | #html_show_sphinx = True 170 | 171 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. 172 | #html_show_copyright = True 173 | 174 | # If true, an OpenSearch description file will be output, and all pages will 175 | # contain a tag referring to it. The value of this option must be the 176 | # base URL from which the finished HTML is served. 177 | #html_use_opensearch = '' 178 | 179 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 180 | #html_file_suffix = None 181 | 182 | # Output file base name for HTML help builder. 183 | htmlhelp_basename = 'bsdploydoc' 184 | 185 | 186 | # -- Options for LaTeX output --------------------------------------------- 187 | 188 | latex_elements = { 189 | # The paper size ('letterpaper' or 'a4paper'). 190 | #'papersize': 'letterpaper', 191 | 192 | # The font size ('10pt', '11pt' or '12pt'). 193 | #'pointsize': '10pt', 194 | 195 | # Additional stuff for the LaTeX preamble. 196 | #'preamble': '', 197 | } 198 | 199 | # Grouping the document tree into LaTeX files. List of tuples 200 | # (source start file, target name, title, 201 | # author, documentclass [howto, manual, or own class]). 202 | latex_documents = [ 203 | ('index', 'bsdploy.tex', u'bsdploy Documentation', 204 | u'Tom Lazar', 'manual'), 205 | ] 206 | 207 | # The name of an image file (relative to this directory) to place at the top of 208 | # the title page. 209 | #latex_logo = None 210 | 211 | # For "manual" documents, if this is true, then toplevel headings are parts, 212 | # not chapters. 213 | #latex_use_parts = False 214 | 215 | # If true, show page references after internal links. 216 | #latex_show_pagerefs = False 217 | 218 | # If true, show URL addresses after external links. 219 | #latex_show_urls = False 220 | 221 | # Documents to append as an appendix to all manuals. 222 | #latex_appendices = [] 223 | 224 | # If false, no module index is generated. 225 | #latex_domain_indices = True 226 | 227 | 228 | # -- Options for manual page output --------------------------------------- 229 | 230 | # One entry per manual page. List of tuples 231 | # (source start file, name, description, authors, manual section). 232 | man_pages = [ 233 | ('index', 'bsdploy', u'bsdploy Documentation', 234 | [u'Tom Lazar'], 1) 235 | ] 236 | 237 | # If true, show URL addresses after external links. 238 | #man_show_urls = False 239 | 240 | 241 | # -- Options for Texinfo output ------------------------------------------- 242 | 243 | # Grouping the document tree into Texinfo files. List of tuples 244 | # (source start file, target name, title, author, 245 | # dir menu entry, description, category) 246 | texinfo_documents = [ 247 | ('index', 'bsdploy', u'bsdploy Documentation', 248 | u'Tom Lazar', 'bsdploy', 'One line description of project.', 249 | 'Miscellaneous'), 250 | ] 251 | 252 | # Documents to append as an appendix to all manuals. 253 | #texinfo_appendices = [] 254 | 255 | # If false, no module index is generated. 256 | #texinfo_domain_indices = True 257 | 258 | # How to display URL addresses: 'footnote', 'no', or 'inline'. 259 | #texinfo_show_urls = 'footnote' 260 | 261 | # If true, do not generate a @detailmenu in the "Top" node's menu. 262 | #texinfo_no_detailmenu = False 263 | -------------------------------------------------------------------------------- /bsdploy/tests/test_quickstart.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pytest 3 | import re 4 | import subprocess 5 | import textwrap 6 | import time 7 | 8 | 9 | @pytest.fixture 10 | def qs_path(): 11 | from bsdploy import bsdploy_path 12 | qs_path = os.path.abspath(os.path.join(bsdploy_path, '..', 'docs', 'quickstart.rst')) 13 | if not os.path.exists(qs_path): 14 | pytest.skip("Can't access quickstart.rst") 15 | return qs_path 16 | 17 | 18 | def strip_block(block): 19 | lines = iter(block) 20 | result = [] 21 | for line in lines: 22 | if not line: 23 | continue 24 | result.append(line) 25 | break 26 | result.extend(lines) 27 | lines = iter(reversed(result)) 28 | result = [] 29 | for line in lines: 30 | if not line: 31 | continue 32 | result.append(line) 33 | break 34 | result.extend(lines) 35 | return textwrap.dedent("\n".join(reversed(result))).split('\n') 36 | 37 | 38 | def iter_blocks(lines): 39 | inindent = False 40 | text = [] 41 | block = [] 42 | for line in lines: 43 | line = line.rstrip() 44 | if inindent and line and line.strip() == line: 45 | inindent = False 46 | text = '\n'.join(text) 47 | text = re.sub('([^\n])\n([^\n])', '\\1\\2', text) 48 | text = re.split('\\n+', text) 49 | yield text, strip_block(block) 50 | text = [] 51 | block = [] 52 | if inindent: 53 | block.append(line) 54 | else: 55 | text.append(line) 56 | if line.endswith('::') or line.startswith('.. code-block::'): 57 | inindent = True 58 | 59 | 60 | def parse_qs(qs_path): 61 | with open(qs_path) as f: 62 | lines = f.read().splitlines() 63 | result = [] 64 | for text, block in iter_blocks(lines): 65 | text = '\n'.join(text) 66 | if block[0].startswith('%'): 67 | result.append(('execute', block)) 68 | elif '``' in text: 69 | names = re.findall('``(.+?)``', text) 70 | if 'create' in text.lower(): 71 | result.append(('create', names, block)) 72 | if 'add' in text.lower(): 73 | result.append(('add', names, block)) 74 | elif 'completed' in text: 75 | result.append(('expect', block)) 76 | return result 77 | 78 | 79 | def iter_quickstart_calls(actions, confext, ployconf, tempdir): 80 | paths = { 81 | 'ploy.conf': ployconf, 82 | 'etc/ploy.conf': ployconf, 83 | 'ploy.yml': ployconf, 84 | 'etc/ploy.yml': ployconf, 85 | 'files.yml': tempdir['bootstrap-files/files.yml'], 86 | 'jailhost.yml': tempdir['host_vars/jailhost.yml'], 87 | 'jailhost-demo_jail.yml': tempdir['jailhost-demo_jail.yml']} 88 | for action in actions: 89 | if action[0] == 'execute': 90 | for line in action[1]: 91 | if line.startswith('%'): 92 | line = line[1:].strip() 93 | parts = line.split() 94 | if len(parts) == 3 and parts[:2] == ['ploy', 'ssh']: 95 | continue 96 | bootstrap = line.endswith('bootstrap') 97 | if bootstrap: 98 | yield (action[0], wait_for_ssh, ('localhost', 44003), {}) 99 | line = '%s -y' % line 100 | yield (action[0], subprocess.check_call, (line,), dict(shell=True)) 101 | if bootstrap: 102 | yield (action[0], wait_for_ssh, ('localhost', 44003), {}) 103 | elif action[0] == 'create': 104 | name = action[1][-1] 105 | content = list(action[2]) 106 | content.append('') 107 | yield (action[0], paths[name].fill, (content,), {}) 108 | elif action[0] == 'add': 109 | name = action[1][-1] 110 | content = paths[name].content().split('\n') 111 | content.extend(action[2]) 112 | content.append('') 113 | yield (action[0], paths[name].fill, (content,), {}) 114 | elif action[0] == 'expect': 115 | pass 116 | else: 117 | pytest.fail("Unknown action %s" % action[0]) 118 | 119 | 120 | def test_quickstart_calls(confext, qs_path, ployconf, tempdir): 121 | calls = [] 122 | for action, func, args, kw in iter_quickstart_calls(parse_qs(qs_path), confext, ployconf, tempdir): 123 | if action in ('add', 'create'): 124 | func(*args, **kw) 125 | calls.append((action, func.__self__.path)) 126 | else: 127 | calls.append((func, args)) 128 | assert calls == [ 129 | (subprocess.check_call, ('pip install "ploy_virtualbox>=2.0.0b1"',)), 130 | (subprocess.check_call, ('mkdir ploy-quickstart',)), 131 | (subprocess.check_call, ('cd ploy-quickstart',)), 132 | (subprocess.check_call, ('mkdir etc',)), 133 | ('create', ('%s/etc/ploy.conf' % tempdir.directory).replace('.conf', confext)), 134 | (subprocess.check_call, ('ploy start ploy-demo',)), 135 | ('add', ('%s/etc/ploy.conf' % tempdir.directory).replace('.conf', confext)), 136 | (wait_for_ssh, ('localhost', 44003)), 137 | (subprocess.check_call, ('ploy bootstrap -y',)), 138 | (wait_for_ssh, ('localhost', 44003)), 139 | ('add', ('%s/etc/ploy.conf' % tempdir.directory).replace('.conf', confext)), 140 | (subprocess.check_call, ('ploy configure jailhost',)), 141 | ('add', ('%s/etc/ploy.conf' % tempdir.directory).replace('.conf', confext)), 142 | (subprocess.check_call, ('ploy start demo_jail',)), 143 | ('create', '%s/jailhost-demo_jail.yml' % tempdir.directory), 144 | (subprocess.check_call, ('ploy configure demo_jail',)), 145 | (subprocess.check_call, ('mkdir host_vars',)), 146 | ('create', '%s/host_vars/jailhost.yml' % tempdir.directory), 147 | (subprocess.check_call, ('ploy configure jailhost -t pf-conf',)), 148 | (subprocess.check_call, ("ploy ssh jailhost 'ifconfig em0'",))] 149 | assert ployconf.content().splitlines() == [ 150 | '[vb-instance:ploy-demo]', 151 | 'vm-nic2 = nat', 152 | 'vm-natpf2 = ssh,tcp,,44003,,22', 153 | 'storage =', 154 | ' --medium vb-disk:defaultdisk', 155 | ' --type dvddrive --medium https://mfsbsd.vx.sk/files/iso/12/amd64/mfsbsd-se-12.0-RELEASE-amd64.iso --medium_sha1 2fbf2be5a79cc8081d918475400581bd54bb30ae', 156 | '', 157 | '[ez-master:jailhost]', 158 | 'instance = ploy-demo', 159 | '', 160 | '[ez-master:jailhost]', 161 | 'instance = ploy-demo', 162 | 'roles =', 163 | ' dhcp_host', 164 | ' jails_host', 165 | '', 166 | '[ez-instance:demo_jail]', 167 | 'ip = 10.0.0.1'] 168 | assert tempdir['jailhost-demo_jail.yml'].content().splitlines() == [ 169 | '---', 170 | '- hosts: jailhost-demo_jail', 171 | ' tasks:', 172 | ' - name: install nginx', 173 | ' pkgng:', 174 | ' name: "nginx"', 175 | ' state: "present"', 176 | ' - name: Setup nginx to start immediately and on boot', 177 | ' service: name=nginx enabled=yes state=started'] 178 | assert tempdir['host_vars/jailhost.yml'].content().splitlines() == [ 179 | 'pf_nat_rules:', 180 | ' - "rdr on em0 proto tcp from any to em0 port 80 -> {{ hostvars[\'jailhost-demo_jail\'][\'ploy_ip\'] }} port 80"'] 181 | 182 | 183 | @pytest.yield_fixture 184 | def virtualenv(monkeypatch, tempdir): 185 | origdir = os.getcwd() 186 | os.chdir(tempdir.directory) 187 | subprocess.check_output(['virtualenv', '.']) 188 | monkeypatch.delenv('PYTHONHOME', raising=False) 189 | monkeypatch.setenv('VIRTUAL_ENV', tempdir.directory) 190 | monkeypatch.setenv('PATH', '%s/bin:%s' % (tempdir.directory, os.environ['PATH'])) 191 | yield tempdir.directory 192 | os.chdir(origdir) 193 | subprocess.call(['VBoxManage', 'controlvm', 'ploy-demo', 'poweroff']) 194 | time.sleep(5) 195 | subprocess.call(['VBoxManage', 'unregistervm', '--delete', 'ploy-demo']) 196 | 197 | 198 | def wait_for_ssh(host, port, timeout=90): 199 | from contextlib import closing 200 | import socket 201 | while timeout > 0: 202 | with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s: 203 | try: 204 | s.settimeout(1) 205 | if s.connect_ex((host, port)) == 0: 206 | if s.recv(128).startswith(b'SSH-2'): 207 | return 208 | except socket.timeout: 209 | timeout -= 1 210 | continue 211 | time.sleep(1) 212 | timeout -= 1 213 | raise RuntimeError( 214 | "SSH at %s:%s didn't become accessible" % (host, port)) 215 | 216 | 217 | @pytest.mark.skipif("not config.option.quickstart_bsdploy") 218 | def test_quickstart_functional(request, qs_path, confext, ployconf, tempdir, virtualenv): 219 | if confext == '.yml': 220 | pytest.xfail("No YML config file support yet") 221 | if not os.path.isabs(request.config.option.quickstart_bsdploy): 222 | pytest.fail("The path given by --quickstart-bsdploy needs to be absolute.") 223 | if request.config.option.ansible_version: 224 | subprocess.check_call(['pip', 'install', 'ansible==%s' % request.config.option.ansible_version]) 225 | else: 226 | subprocess.check_call(['pip', 'install', 'ansible']) 227 | subprocess.check_call(['pip', 'install', '-i' 'https://d.rzon.de:8141/fschulze/dev/', '--pre', request.config.option.quickstart_bsdploy]) 228 | for action, func, args, kw in iter_quickstart_calls(parse_qs(qs_path), confext, ployconf, tempdir): 229 | func(*args, **kw) 230 | -------------------------------------------------------------------------------- /bsdploy/tests/test_bootstrap_mfsbsd.py: -------------------------------------------------------------------------------- 1 | from bsdploy import bsdploy_path 2 | import pytest 3 | 4 | 5 | @pytest.fixture 6 | def bootstrap(env_mock, environ_mock, monkeypatch, put_mock, run_mock, tempdir, yesno_mock, ployconf): 7 | from bsdploy.fabfile_mfsbsd import bootstrap 8 | ployconf.fill('') 9 | environ_mock['HOME'] = tempdir.directory 10 | monkeypatch.setattr('bsdploy.fabfile_mfsbsd.env', env_mock) 11 | monkeypatch.setattr('bsdploy.fabfile_mfsbsd.run', run_mock) 12 | monkeypatch.setattr('bsdploy.fabfile_mfsbsd.yesno', yesno_mock) 13 | return bootstrap 14 | 15 | 16 | def create_ssh_host_keys(tempdir): 17 | tempdir['default-test_instance/bootstrap-files/ssh_host_dsa_key'].fill('dsa') 18 | tempdir['default-test_instance/bootstrap-files/ssh_host_dsa_key.pub'].fill('dsa.pub') 19 | tempdir['default-test_instance/bootstrap-files/ssh_host_ecdsa_key'].fill('ecdsa') 20 | tempdir['default-test_instance/bootstrap-files/ssh_host_ecdsa_key.pub'].fill('ecdsa.pub') 21 | tempdir['default-test_instance/bootstrap-files/ssh_host_ed25519_key'].fill('ed25519') 22 | tempdir['default-test_instance/bootstrap-files/ssh_host_ed25519_key.pub'].fill('ed25519.pub') 23 | tempdir['default-test_instance/bootstrap-files/ssh_host_rsa_key'].fill('rsa') 24 | tempdir['default-test_instance/bootstrap-files/ssh_host_rsa_key.pub'].fill('rsa.pub') 25 | 26 | 27 | def test_bootstrap_ask_to_continue(bootstrap, capsys, default_mounts, run_mock, run_result, tempdir, yesno_mock): 28 | format_info = dict( 29 | bsdploy_path=bsdploy_path, 30 | tempdir=tempdir.directory) 31 | tempdir['default-test_instance/bootstrap-files/authorized_keys'].fill('id_dsa') 32 | create_ssh_host_keys(tempdir) 33 | run_mock.expected = [ 34 | ('mount', {}, default_mounts), 35 | ('test -e /dev/cd0 && mount_cd9660 /dev/cd0 /cdrom || true', {}, '\n'), 36 | ('test -e /dev/da0a && mount -o ro /dev/da0a /media || true', {}, '\n'), 37 | ("find /cdrom/ /media/ -name 'base.txz' -exec dirname {} \\;", {}, run_result('/cdrom/9.2-RELEASE-amd64', 0)), 38 | ('sysctl -n hw.realmem', {}, '536805376'), 39 | ('sysctl -n kern.disks', {}, 'ada0 cd0\n'), 40 | ('ifconfig -l', {}, 'em0 lo0')] 41 | yesno_mock.expected = [ 42 | ("\nContinuing will destroy the existing data on the following devices:\n ada0\n\nContinue?", False)] 43 | bootstrap() 44 | (out, err) = capsys.readouterr() 45 | out_lines = out.splitlines() 46 | assert out_lines == [ 47 | "", 48 | "Using these local files for bootstrapping:", 49 | "%(bsdploy_path)s/bootstrap-files/FreeBSD.conf -(template:True)-> /mnt/usr/local/etc/pkg/repos/FreeBSD.conf" % format_info, 50 | "%(tempdir)s/default-test_instance/bootstrap-files/authorized_keys -(template:False)-> /mnt/root/.ssh/authorized_keys" % format_info, 51 | "%(bsdploy_path)s/bootstrap-files/make.conf -(template:False)-> /mnt/etc/make.conf" % format_info, 52 | "%(bsdploy_path)s/bootstrap-files/pf.conf -(template:False)-> /mnt/etc/pf.conf" % format_info, 53 | "%(bsdploy_path)s/bootstrap-files/pkg.conf -(template:True)-> /mnt/usr/local/etc/pkg.conf" % format_info, 54 | "%(bsdploy_path)s/bootstrap-files/rc.conf -(template:True)-> /mnt/etc/rc.conf" % format_info, 55 | "%(tempdir)s/default-test_instance/bootstrap-files/ssh_host_dsa_key -(template:False)-> /mnt/etc/ssh/ssh_host_dsa_key" % format_info, 56 | "%(tempdir)s/default-test_instance/bootstrap-files/ssh_host_dsa_key.pub -(template:False)-> /mnt/etc/ssh/ssh_host_dsa_key.pub" % format_info, 57 | "%(tempdir)s/default-test_instance/bootstrap-files/ssh_host_ecdsa_key -(template:False)-> /mnt/etc/ssh/ssh_host_ecdsa_key" % format_info, 58 | "%(tempdir)s/default-test_instance/bootstrap-files/ssh_host_ecdsa_key.pub -(template:False)-> /mnt/etc/ssh/ssh_host_ecdsa_key.pub" % format_info, 59 | "%(tempdir)s/default-test_instance/bootstrap-files/ssh_host_ed25519_key -(template:False)-> /mnt/etc/ssh/ssh_host_ed25519_key" % format_info, 60 | "%(tempdir)s/default-test_instance/bootstrap-files/ssh_host_ed25519_key.pub -(template:False)-> /mnt/etc/ssh/ssh_host_ed25519_key.pub" % format_info, 61 | "%(tempdir)s/default-test_instance/bootstrap-files/ssh_host_rsa_key -(template:False)-> /mnt/etc/ssh/ssh_host_rsa_key" % format_info, 62 | "%(tempdir)s/default-test_instance/bootstrap-files/ssh_host_rsa_key.pub -(template:False)-> /mnt/etc/ssh/ssh_host_rsa_key.pub" % format_info, 63 | "%(bsdploy_path)s/bootstrap-files/sshd_config -(template:False)-> /mnt/etc/ssh/sshd_config" % format_info, 64 | "", 65 | "", 66 | "Found the following disk devices on the system:", 67 | " ada0 cd0", 68 | "", 69 | "Found the following network interfaces, now is your chance to update your rc.conf accordingly!", 70 | " em0", 71 | "", 72 | 'The generated rc_conf:', 73 | 'hostname="test_instance"', 74 | 'sshd_enable="YES"', 75 | 'syslogd_flags="-ss"', 76 | 'zfs_enable="YES"', 77 | 'pf_enable="YES"', 78 | 'ifconfig_em0="DHCP"', 79 | '', 80 | 'bootstrap-bsd-url: /cdrom/9.2-RELEASE-amd64', 81 | 'bootstrap-system-pool-name: system', 82 | 'bootstrap-data-pool-name: tank', 83 | 'bootstrap-swap-size: 1024M', 84 | 'bootstrap-system-pool-size: 20G', 85 | 'bootstrap-firstboot-update: False', 86 | 'bootstrap-autoboot-delay: -1', 87 | 'bootstrap-reboot: True', 88 | 'bootstrap-packages: python27', 89 | '', 90 | "Continuing will destroy the existing data on the following devices:", 91 | " ada0", 92 | "", 93 | "Continue?"] 94 | 95 | 96 | def test_bootstrap_no_newline_at_end_of_rc_conf(bootstrap, capsys, default_mounts, local_mock, run_mock, run_result, tempdir): 97 | tempdir['default-test_instance/bootstrap-files/authorized_keys'].fill('id_dsa') 98 | create_ssh_host_keys(tempdir) 99 | tempdir['default-test_instance/bootstrap-files/rc.conf'].fill('foo', allow_conf=True) 100 | run_mock.expected = [ 101 | ('mount', {}, default_mounts), 102 | ('test -e /dev/cd0 && mount_cd9660 /dev/cd0 /cdrom || true', {}, '\n'), 103 | ('test -e /dev/da0a && mount -o ro /dev/da0a /media || true', {}, '\n'), 104 | ("find /cdrom/ /media/ -name 'base.txz' -exec dirname {} \\;", {}, run_result('/cdrom/9.2-RELEASE-amd64', 0)), 105 | ('sysctl -n hw.realmem', {}, '536805376'), 106 | ('sysctl -n kern.disks', {}, 'ada0 cd0\n'), 107 | ('ifconfig -l', {}, 'em0 lo0')] 108 | bootstrap() 109 | (out, err) = capsys.readouterr() 110 | out_lines = out.splitlines() 111 | assert out_lines[-4:] == [ 112 | "ERROR! Your rc.conf doesn't end in a newline:", 113 | '==========', 114 | 'foo<<<<<<<<<<', 115 | ''] 116 | 117 | 118 | def test_bootstrap(bootstrap, default_mounts, put_mock, run_mock, run_result, tempdir, yesno_mock): 119 | format_info = dict( 120 | bsdploy_path=bsdploy_path, 121 | tempdir=tempdir.directory) 122 | tempdir['default-test_instance/bootstrap-files/authorized_keys'].fill('id_dsa') 123 | create_ssh_host_keys(tempdir) 124 | put_mock.expected = [ 125 | ((object, '/mnt/usr/local/etc/pkg/repos/FreeBSD.conf'), {'mode': None}), 126 | (("%(tempdir)s/default-test_instance/bootstrap-files/authorized_keys" % format_info, '/mnt/root/.ssh/authorized_keys'), {'mode': None}), 127 | (("%(bsdploy_path)s/bootstrap-files/make.conf" % format_info, '/mnt/etc/make.conf'), {'mode': None}), 128 | # put from upload_template 129 | ((object, '/mnt/etc/pf.conf'), {'mode': None}), 130 | ((object, '/mnt/usr/local/etc/pkg.conf'), {'mode': None}), 131 | ((object, '/mnt/etc/rc.conf'), {'mode': None}), 132 | (("%(tempdir)s/default-test_instance/bootstrap-files/ssh_host_dsa_key" % format_info, '/mnt/etc/ssh/ssh_host_dsa_key'), {'mode': 0o600}), 133 | (("%(tempdir)s/default-test_instance/bootstrap-files/ssh_host_dsa_key.pub" % format_info, '/mnt/etc/ssh/ssh_host_dsa_key.pub'), {'mode': 0o644}), 134 | (("%(tempdir)s/default-test_instance/bootstrap-files/ssh_host_ecdsa_key" % format_info, '/mnt/etc/ssh/ssh_host_ecdsa_key'), {'mode': 0o600}), 135 | (("%(tempdir)s/default-test_instance/bootstrap-files/ssh_host_ecdsa_key.pub" % format_info, '/mnt/etc/ssh/ssh_host_ecdsa_key.pub'), {'mode': 0o644}), 136 | (("%(tempdir)s/default-test_instance/bootstrap-files/ssh_host_ed25519_key" % format_info, '/mnt/etc/ssh/ssh_host_ed25519_key'), {'mode': 0o600}), 137 | (("%(tempdir)s/default-test_instance/bootstrap-files/ssh_host_ed25519_key.pub" % format_info, '/mnt/etc/ssh/ssh_host_ed25519_key.pub'), {'mode': 0o644}), 138 | (("%(tempdir)s/default-test_instance/bootstrap-files/ssh_host_rsa_key" % format_info, '/mnt/etc/ssh/ssh_host_rsa_key'), {'mode': 0o600}), 139 | (("%(tempdir)s/default-test_instance/bootstrap-files/ssh_host_rsa_key.pub" % format_info, '/mnt/etc/ssh/ssh_host_rsa_key.pub'), {'mode': 0o644}), 140 | (("%(bsdploy_path)s/bootstrap-files/sshd_config" % format_info, '/mnt/etc/ssh/sshd_config'), {'mode': None}), 141 | ] 142 | run_mock.expected = [ 143 | ('mount', {}, default_mounts), 144 | ('test -e /dev/cd0 && mount_cd9660 /dev/cd0 /cdrom || true', {}, '\n'), 145 | ('test -e /dev/da0a && mount -o ro /dev/da0a /media || true', {}, '\n'), 146 | ("find /cdrom/ /media/ -name 'base.txz' -exec dirname {} \\;", {}, run_result('/cdrom/9.2-RELEASE-amd64', 0)), 147 | ('sysctl -n hw.realmem', {}, '536805376'), 148 | ('sysctl -n kern.disks', {}, 'ada0 cd0\n'), 149 | ('ifconfig -l', {}, 'em0 lo0'), 150 | ('destroygeom -d ada0 -p system -p tank', {}, ''), 151 | ('zfsinstall -d ada0 -p system -V 28 -u /cdrom/9.2-RELEASE-amd64 -s 1024M -z 20G', {'shell': False}, ''), 152 | ('gpart add -t freebsd-zfs -l tank_ada0 ada0', {}, ''), 153 | ('cp /etc/resolv.conf /mnt/etc/resolv.conf', {'warn_only': True}, ''), 154 | ('mkdir -p "/mnt/usr/local/etc/pkg/repos"', {'shell': False}, ''), 155 | ('mkdir -p "/mnt/root/.ssh" && chmod 0600 "/mnt/root/.ssh"', {'shell': False}, ''), 156 | ('chroot /mnt pkg update', {'shell': False}, ''), 157 | ('chroot /mnt pkg install python27', {'shell': False}, ''), 158 | ('echo autoboot_delay=-1 >> /mnt/boot/loader.conf', {}, ''), 159 | ('reboot', {}, '')] 160 | yesno_mock.expected = [ 161 | ("\nContinuing will destroy the existing data on the following devices:\n ada0\n\nContinue?", True)] 162 | bootstrap() 163 | -------------------------------------------------------------------------------- /bsdploy/tests/test_bsdploy.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | import os 3 | import pytest 4 | 5 | 6 | @pytest.fixture 7 | def ctrl(ployconf, tempdir): 8 | from ploy import Controller 9 | import bsdploy 10 | import ploy.plain 11 | import ploy_ezjail 12 | import ploy_fabric 13 | ployconf.fill([ 14 | '[ez-master:jailhost]', 15 | '[instance:foo]', 16 | 'master = jailhost']) 17 | ctrl = Controller(configpath=ployconf.directory) 18 | ctrl.plugins = { 19 | 'bsdploy': bsdploy.plugin, 20 | 'ezjail': ploy_ezjail.plugin, 21 | 'fabric': ploy_fabric.plugin, 22 | 'plain': ploy.plain.plugin} 23 | ctrl.configfile = ployconf.path 24 | return ctrl 25 | 26 | 27 | def test_bootstrap_command(capsys, ctrl, monkeypatch): 28 | def do(self, *args, **kwargs): 29 | print("do with %r, %r called!" % (args, kwargs)) 30 | monkeypatch.setattr('ploy_fabric.do', do) 31 | ctrl(['./bin/ploy', 'bootstrap']) 32 | (out, err) = capsys.readouterr() 33 | out_lines = out.splitlines() 34 | assert out_lines == [ 35 | "do with ('bootstrap',), {'bootstrap-yes': False} called!"] 36 | 37 | 38 | def test_augment_ezjail_master(ctrl, ployconf, tempdir): 39 | tempdir['jailhost/bootstrap-files/ssh_host_rsa_key.pub'].fill('rsa') 40 | config = dict(ctrl.instances['jailhost'].config) 41 | assert sorted(config.keys()) == [ 42 | 'ansible_python_interpreter', 'fabfile', 'fabric-shell', 43 | 'roles', 'ssh-host-keys'] 44 | assert config['ansible_python_interpreter'] == '/usr/local/bin/python2.7' 45 | assert config['fabfile'].endswith('fabfile_mfsbsd.py') 46 | assert config['fabric-shell'] == '/bin/sh -c' 47 | assert config['ssh-host-keys'].endswith('bootstrap-files/ssh_host_rsa_key.pub') 48 | assert os.path.exists(config['fabfile']), "The fabfile '%s' doesn't exist." % config['fabfile'] 49 | assert config['roles'] == 'jails_host' 50 | 51 | 52 | def test_augment_ezjail_master_playbook_implicit(ctrl, ployconf, tempdir): 53 | from ploy_ansible import has_playbook 54 | jailhost_yml = tempdir['jailhost.yml'] 55 | jailhost_yml.fill('') 56 | config = dict(ctrl.instances['jailhost'].config) 57 | assert 'roles' not in config 58 | assert 'playbook' not in config 59 | assert has_playbook(ctrl.instances['jailhost']) 60 | 61 | 62 | def test_augment_ezjail_master_playbook_explicit(ctrl, ployconf, tempdir): 63 | from ploy_ansible import has_playbook 64 | ployconf.fill([ 65 | '[ez-master:jailhost]', 66 | 'playbook = blubber.yml']) 67 | config = dict(ctrl.instances['jailhost'].config) 68 | assert 'roles' not in config 69 | assert config['playbook'] == 'blubber.yml' 70 | assert has_playbook(ctrl.instances['jailhost']) 71 | 72 | 73 | def test_augment_ezjail_master_fabfile_default_mfsbsd(ctrl, ployconf, tempdir): 74 | """ if no fabfile is stated and the by-convention does not exist, 75 | the default is set """ 76 | config = dict(ctrl.instances['jailhost'].config) 77 | assert config['fabfile'].endswith('fabfile_mfsbsd.py') 78 | from ploy_fabric import get_fabfile 79 | assert get_fabfile(ctrl.instances['jailhost']).endswith('fabfile_mfsbsd.py') 80 | 81 | 82 | def test_augment_ezjail_master_fabfile_implicit(ctrl, ployconf, tempdir): 83 | jailhost_fab = tempdir['jailhost/fabfile.py'] 84 | jailhost_fab.fill('') 85 | config = dict(ctrl.instances['jailhost'].config) 86 | assert config['fabfile'].endswith('jailhost/fabfile.py') 87 | from ploy_fabric import get_fabfile 88 | assert get_fabfile(ctrl.instances['jailhost']).endswith('jailhost/fabfile.py') 89 | 90 | 91 | def test_augment_ezjail_jail_fabfile_implicit(ctrl, ployconf, tempdir): 92 | jailhost_fab = tempdir['jailhost-foo/fabfile.py'] 93 | jailhost_fab.fill('') 94 | config = dict(ctrl.instances['foo'].config) 95 | assert config['fabfile'].endswith('jailhost-foo/fabfile.py') 96 | from ploy_fabric import get_fabfile 97 | assert get_fabfile(ctrl.instances['foo']).endswith('jailhost-foo/fabfile.py') 98 | 99 | 100 | def test_augment_ezjail_master_fabfile_explicit(ctrl, ployconf, tempdir): 101 | jailhost_fab = tempdir['jailhost/fabfile.py'] 102 | jailhost_fab.fill('') 103 | jailhost_fab = tempdir['blubber.py'] 104 | jailhost_fab.fill('') 105 | ployconf.fill([ 106 | '[ez-master:jailhost]', 107 | 'fabfile = ../blubber.py']) 108 | config = dict(ctrl.instances['jailhost'].config) 109 | assert config['fabfile'].endswith('blubber.py') 110 | from ploy_fabric import get_fabfile 111 | assert get_fabfile(ctrl.instances['jailhost']).endswith('blubber.py') 112 | 113 | 114 | def test_augment_ezjail_instance(ctrl, ployconf): 115 | config = dict(ctrl.instances['foo'].config) 116 | assert sorted(config.keys()) == [ 117 | 'ansible_python_interpreter', 'fabric-shell', 'flavour', 'master', 118 | 'startup_script'] 119 | assert config['ansible_python_interpreter'] == '/usr/local/bin/python2.7' 120 | assert config['fabric-shell'] == '/bin/sh -c' 121 | assert config['flavour'] == 'bsdploy_base' 122 | assert config['master'] == 'jailhost' 123 | assert config['startup_script']['path'].endswith('startup-ansible-jail.sh') 124 | assert os.path.exists(config['startup_script']['path']), "The startup_script at '%s' doesn't exist." % config['startup_script']['path'] 125 | 126 | 127 | def test_augment_non_ezjail_instance(ctrl, ployconf): 128 | ployconf.fill([ 129 | '[plain-instance:foo]']) 130 | assert dict(ctrl.instances['foo'].config) == {} 131 | 132 | 133 | @pytest.mark.parametrize("config, expected", [ 134 | ([], {'vboxnet0': {'ip': '192.168.56.1'}}), 135 | (['[vb-hostonlyif:vboxnet0]'], {'vboxnet0': {'ip': '192.168.56.1'}}), 136 | (['[vb-hostonlyif:vboxnet0]', 'ip = 192.168.57.1'], 137 | {'vboxnet0': {'ip': '192.168.57.1'}}), 138 | (['[vb-hostonlyif:vboxnet1]', 'ip = 192.168.57.1'], 139 | {'vboxnet0': {'ip': '192.168.56.1'}, 'vboxnet1': {'ip': '192.168.57.1'}})]) 140 | def test_virtualbox_hostonlyif(ctrl, config, expected, ployconf): 141 | import ploy_virtualbox 142 | ctrl.plugins['virtualbox'] = ploy_virtualbox.plugin 143 | ployconf.fill([ 144 | '[vb-instance:vb]'] + config) 145 | # trigger augmentation 146 | ctrl.instances['vb'] 147 | assert ctrl.config['vb-hostonlyif'] == expected 148 | 149 | 150 | @pytest.mark.parametrize("config, expected", [ 151 | ([], 152 | {'vboxnet0': { 153 | 'ip': '192.168.56.2', 'netmask': '255.255.255.0', 154 | 'lowerip': '192.168.56.100', 'upperip': '192.168.56.254'}}), 155 | (['[vb-dhcpserver:vboxnet0]'], 156 | {'vboxnet0': { 157 | 'ip': '192.168.56.2', 'netmask': '255.255.255.0', 158 | 'lowerip': '192.168.56.100', 'upperip': '192.168.56.254'}}), 159 | (['[vb-dhcpserver:vboxnet0]', 'ip = 192.168.57.2'], 160 | {'vboxnet0': { 161 | 'ip': '192.168.57.2', 'netmask': '255.255.255.0', 162 | 'lowerip': '192.168.56.100', 'upperip': '192.168.56.254'}}), 163 | (['[vb-dhcpserver:vboxnet0]', 'netmask = 255.255.0.0'], 164 | {'vboxnet0': { 165 | 'ip': '192.168.56.2', 'netmask': '255.255.0.0', 166 | 'lowerip': '192.168.56.100', 'upperip': '192.168.56.254'}}), 167 | (['[vb-dhcpserver:vboxnet0]', 'lowerip = 192.168.56.50'], 168 | {'vboxnet0': { 169 | 'ip': '192.168.56.2', 'netmask': '255.255.255.0', 170 | 'lowerip': '192.168.56.50', 'upperip': '192.168.56.254'}}), 171 | (['[vb-dhcpserver:vboxnet0]', 'upperip = 192.168.56.200'], 172 | {'vboxnet0': { 173 | 'ip': '192.168.56.2', 'netmask': '255.255.255.0', 174 | 'lowerip': '192.168.56.100', 'upperip': '192.168.56.200'}}), 175 | (['[vb-dhcpserver:vboxnet0]', 'ip = 192.168.57.2', 'netmask = 255.255.0.0', 'lowerip = 192.168.56.50', 'upperip = 192.168.56.200'], 176 | {'vboxnet0': { 177 | 'ip': '192.168.57.2', 'netmask': '255.255.0.0', 178 | 'lowerip': '192.168.56.50', 'upperip': '192.168.56.200'}}), 179 | (['[vb-dhcpserver:vboxnet1]', 'ip = 192.168.57.2'], 180 | {'vboxnet0': { 181 | 'ip': '192.168.56.2', 'netmask': '255.255.255.0', 182 | 'lowerip': '192.168.56.100', 'upperip': '192.168.56.254'}, 183 | 'vboxnet1': { 184 | 'ip': '192.168.57.2'}})]) 185 | def test_virtualbox_dhcpserver(ctrl, config, expected, ployconf): 186 | import ploy_virtualbox 187 | ctrl.plugins['virtualbox'] = ploy_virtualbox.plugin 188 | ployconf.fill([ 189 | '[vb-instance:vb]'] + config) 190 | # trigger augmentation 191 | ctrl.instances['vb'] 192 | assert ctrl.config['vb-dhcpserver'] == expected 193 | 194 | 195 | @pytest.mark.parametrize("config, expected", [ 196 | ([], {}), 197 | (['storage = --medium vb-disk:defaultdisk'], 198 | {'defaultdisk': {'size': '102400'}}), 199 | (['storage = --medium vb-disk:defaultdisk', '[vb-disk:defaultdisk]'], 200 | {'defaultdisk': {'size': '102400'}}), 201 | (['storage = --medium vb-disk:defaultdisk', '[vb-disk:defaultdisk]', 'size = 1024000'], 202 | {'defaultdisk': {'size': '1024000'}}), 203 | (['storage = --medium vb-disk:defaultdisk', '[vb-disk:otherdisk]'], 204 | {'defaultdisk': {'size': '102400'}, 205 | 'otherdisk': {}})]) 206 | def test_virtualbox_defaultdisk(ctrl, config, expected, ployconf): 207 | import ploy_virtualbox 208 | ctrl.plugins['virtualbox'] = ploy_virtualbox.plugin 209 | ployconf.fill([ 210 | '[vb-instance:vb]'] + config) 211 | # trigger augmentation 212 | ctrl.instances['vb'] 213 | assert ctrl.config.get('vb-disk', {}) == expected 214 | 215 | 216 | @pytest.mark.parametrize("instance, key, value, expected", [ 217 | ('foo', 'ansible_python_interpreter', 'python2.7', 'python2.7'), 218 | ('foo', 'startup_script', 'foo', {'path': '{tempdir}/etc/foo'}), 219 | ('foo', 'flavour', 'foo', 'foo'), 220 | ('jailhost', 'bootstrap', 'daemonology', ('fabfile', '{bsdploy_path}/fabfile_daemonology.py')), 221 | ('jailhost', 'bootstrap', 'foo', SystemExit), 222 | ('jailhost', 'fabfile', 'fabfile.py', '{tempdir}/etc/fabfile.py'), 223 | ('jailhost', 'fabfile', 'fab1file.py', SystemExit)]) 224 | def test_augment_overwrite(ctrl, instance, key, value, expected, ployconf, tempdir): 225 | from bsdploy import bsdploy_path 226 | ployconf.fill([ 227 | '[ez-master:jailhost]', 228 | '%s = %s' % (key, value), 229 | '[instance:foo]', 230 | 'master = jailhost', 231 | '%s = %s' % (key, value)]) 232 | tempdir['etc/fabfile.py'].fill('') 233 | try: 234 | config = dict(ctrl.instances[instance].config) 235 | except BaseException as e: 236 | assert type(e) == expected 237 | else: 238 | format_info = dict( 239 | bsdploy_path=bsdploy_path, 240 | tempdir=tempdir.directory) 241 | if isinstance(expected, tuple): 242 | key, expected = expected 243 | if isinstance(expected, dict): 244 | expected = { 245 | k: v.format(**format_info) 246 | for k, v in expected.items()} 247 | else: 248 | expected = expected.format(**format_info) 249 | assert config[key] == expected 250 | -------------------------------------------------------------------------------- /bsdploy/fabfile_mfsbsd.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | from __future__ import print_function 3 | from bsdploy.bootstrap_utils import BootstrapUtils 4 | from contextlib import contextmanager 5 | from fabric.api import env, hide, run, settings, task 6 | from ploy.common import yesno 7 | from ploy.config import value_asbool 8 | 9 | # a plain, default fabfile for jailhosts using mfsbsd 10 | 11 | 12 | env.shell = '/bin/sh -c' 13 | 14 | 15 | @contextmanager 16 | def _mfsbsd(env, kwargs={}): 17 | old_shell = env.get('shell') 18 | old_config = env.instance.config.copy() 19 | try: 20 | env.shell = '/bin/sh -c' 21 | 22 | # default ssh settings for mfsbsd with possible overwrite by bootstrap-ssh-host-keys 23 | env.instance.config['ssh-host-keys'] = env.instance.config.get( 24 | 'bootstrap-ssh-host-keys', 25 | '\n'.join([ 26 | # mfsbsd 10.3 27 | 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDnxIsRrqK2Zj73DPB3doYO8eDue2mVcae9oQNAwGz1o7VBmOpAZiscxOz1kg/M/CD3VRchgT5OcbciqJGaWeNyZHzHbVpIzUCycSI28WVpG7B4jXZTcq6vGGBpD22Ms6rTczigEJmshVR3rNxHmswwImmEwR6o1KVRCOAY2gL8Ik6OOKAqWqY8mstx059MsY9usDl2FDn57T8fZ4QMd+DQBEKwhkhqHs8n2WSlJlZqCuWDBNDH0RskDizrZRz+g4ciRwAM5e2dzgaOvtlfT42WD1kxwJIVFJi/1R0O+Xw2/kGyRweJXCqdUbfynFaTm1yen+IUPzNH/jBMtxUiL25r', 28 | # mfsbsd 10.3 se 29 | 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCqSVYJPcXOqPEv/RYV5WiDbr9K/Bz5OeU2Hayo+oBMkxwFuv9KSZGHmZ/EbJOVKhdjDtRDgenxluLU6d5F/vWyGK1M1rdzEFuWfUdfe5Htvz1KEgj/nY5x8OC1h5xme1OwCcFF7oAf7GV6YQtsKF0CZoGwSJEuGb988r8le0VqKy/u4nRiTH+pLHcZzgx6khIl1ty+mBTLgAC7tTgXhB7l83lr/HqU+ZLWZbNohbdEdDWJYVdWHWVMdETc6PG8/DISNfdKuq3YfDyQ/0uZ/uGMJKr7y/J/cabi5VRdVZvdqqbEPLW2zjDtXtRh6+yE4NZETSYx+Wu/DZcn8OsR9pr/', 30 | # mfsbsd 11.1 31 | 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGV77dtuCFh+JXk3gewNaCaW7imdDIXU1xQoW0nmYXJorzEwS8iSEbsbZxN/h8u8nTumPipNy6JeTS21CHIC19A=', 32 | 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBm28fXoW82ISHhm+T4EXc5QU5Fq0tyreJa79UohGwvw', 33 | 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDY+mdQA1uESkvs6V/6SLs+g/8ge9om35ehSzyfZgDZldw5EXlimWzMsI2/eZHgyUJQIZkuP7V/otrWAX5D4K9paqcbUUnDkSenB27VcLFzNq67xWL3kevYMNFFb1Mfg4l9Yiq5mOO8mWDveuQlAR+0CqVo6wOAmCw857x0/raeBruh56zU6i64927sW745BQruFNd+beBW/Sr5yuML8yXWu1LoWOrc+MGVfxbGTcbNEu4CE4voYkxZ9uX+KhSkUO0Sg5fxOrxUDrNeY2oJB74os3WafHuODUaPrXNGOkBpSxF5B/BbqgfCX3H/GQ2gl3NvKT9eRzfwwyoks567Kv1f', 34 | # mfsbsd 11.1 se 35 | 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEvG5hTrYKjRtaIrQED+jd/y/Craqzz/11+ky1P/lyv/e8NH2iX9iPKcVLNQa4G9z2aOBVLFc3GQSyySuFlAB/Q=', 36 | 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAuuvF9hyDyop83H/zYLDqimiR18X+NsAU9UpoDOGHVZ', 37 | 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDAI6Yj/xqZt1yrtgsOHv45mfS8gVPh5IVK+n6hjmnV+RlbwFksA7lFsjY8SRasJko4iqJmxRSmT1bJ/fuOegqFaEa2LwSaEM/J7rx9lHroKO6rtx81Ja54IY8bRscNxaxl+LFFw8F0v4F9hIfzhooLCXVaLgH/y0ScW7gjft9J6omUcfZPIvdMJuOYIHIRqLjL+HnmZSZEh0GWpMraCts3ud+na2gcHuWYMmUpbEeQIkG3FsgTsLlpkrpQLApo7fHNo1FxIbufiopdQ4zDDQFNZod9jRbV1GwUVTAHot6uOZg+oxnCKnHriKaY2/N8QISkVDsEMmGR9Ib5eQkJK9Mv', 38 | # mfsbsd 12.0 39 | 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBM0Y/x3g2rByYjkN8oxuHDZV2VgNCqzrZY41QzNPG+FHBbJbNlhq4zjfj550RxxefwZySWkFHfHHnBOmICihRS8=', 40 | 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPRcWIiTYNuxkh6pIAvULxFoYXmhqsDvMWzDqRKNpC7Z', 41 | 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDBq/bd0ioNAKwVCN+p5xn4bGdg3QzN3Jqw/OyzykMT08XRSIvfTRO0TzqEKF/wDyOQCm+V6Dm0Wx0/Ybg0lxf12az/sQrRPS8VmviJmcxqOIYSLG4G//7Kn+kkAUarXS8L0NdPyon1eT63tpX2twWiawmasWpkkJS4VFt8c4Obj+96AwFW8N9sLlk6iFskj+hdAAUVZjhy8TzPkBzHY5Cljnwui5vz6RVagX9/fkJHuCSFHrGZ/ouuTJCq0S91cr75fWCifINrGSurOaFv7hAc/7l689qLlfZ4Jc8Lxt5ZTeQOMTYMoKLX4lmCWC8mgCvASzn544kLGMUHC4AkrbcD', 42 | # mfsbsd 12.0 se 43 | 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNJR/RUd2QKYgcBY9987ymlLBGUsQe22A/W9NtJ0P+WPFbtigbcESxi2fjZS2tOw2BRS85r9dxCSxNGlwYjw09o=', 44 | 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICtdH4RvvStuu51nq8oiHRbyBB6UUISEA2iyMbg2t4IO', 45 | 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQD28UisgIiBqrlR+47V8v6ek5+fe58iaVIzLvsrEWDREh8QkSR01ZfxXb70oet/5hbRS5Fnnc1evw+5lNLAj3xHN0B1nGL4u3mUdoZX7w3I7llHG6A77Y0UmdgA9GF4xAxSRp75Cv5Ru7AQ4yczIc3J7KKjQgVwGFEdsbUnKENao4+yoHsFOG3eX63Zoqkxv1DphUfVT04IaUf6eyoJBOGmVhplDMWBIxkDG54JiFl/8CjyMEWpnYotmFsDfgfLkgmeyHad+lCvBsEM44QtZGO4F4nko8eFhOH2pwpDeczpgboC3CNjvuHW4ubp/6NUX+IAb812g8+IoCRafCyar1G5'])) 46 | env.instance.config.setdefault('password-fallback', True) 47 | env.instance.config.setdefault('password', 'mfsroot') 48 | if 'bootstrap-ssh-fingerprints' in env.instance.config: 49 | env.instance.config['ssh-fingerprints'] = env.instance.config['bootstrap-ssh-fingerprints'] 50 | # allow overwrites from the commandline 51 | env.instance.config.update(kwargs) 52 | 53 | yield 54 | finally: 55 | added_keys = set(env.instance.config) - set(old_config) 56 | for key in added_keys: 57 | del env.instance.config[key] 58 | env.instance.config.update(old_config) 59 | if old_shell is None: 60 | del env['shell'] 61 | else: 62 | env.shell = old_shell 63 | 64 | 65 | def _bootstrap(): 66 | if 'bootstrap-password' in env.instance.config: 67 | print("Using password for bootstrap SSH connection") 68 | env.instance.config['password-fallback'] = True 69 | env.instance.config['password'] = env.instance.config['bootstrap-password'] 70 | bu = BootstrapUtils() 71 | bu.generate_ssh_keys() 72 | bu.print_bootstrap_files() 73 | # gather infos 74 | if not bu.bsd_url: 75 | print("Found no FreeBSD system to install, please use 'special edition' or specify bootstrap-bsd-url and make sure mfsbsd is running") 76 | return 77 | # get realmem here, because it may fail and we don't want that to happen 78 | # in the middle of the bootstrap 79 | realmem = bu.realmem 80 | print("\nFound the following disk devices on the system:\n %s" % ' '.join(bu.sysctl_devices)) 81 | if bu.first_interface: 82 | print("\nFound the following network interfaces, now is your chance to update your rc.conf accordingly!\n %s" % ' '.join(bu.phys_interfaces)) 83 | else: 84 | print("\nWARNING! Found no suitable network interface!") 85 | 86 | template_context = {"ploy_jail_host_pkg_repository": "pkg+http://pkg.freeBSD.org/${ABI}/quarterly"} 87 | # first the config, so we don't get something essential overwritten 88 | template_context.update(env.instance.config) 89 | template_context.update( 90 | devices=bu.sysctl_devices, 91 | interfaces=bu.phys_interfaces, 92 | hostname=env.instance.id) 93 | 94 | rc_conf = bu.bootstrap_files['rc.conf'].read(template_context).decode('utf-8') 95 | if not rc_conf.endswith('\n'): 96 | print("\nERROR! Your rc.conf doesn't end in a newline:\n==========\n%s<<<<<<<<<<\n" % rc_conf) 97 | return 98 | rc_conf_lines = rc_conf.split('\n') 99 | 100 | for interface in [bu.first_interface, env.instance.config.get('ansible-dhcp_host_sshd_interface')]: 101 | if interface is None: 102 | continue 103 | ifconfig = 'ifconfig_%s' % interface 104 | for line in rc_conf_lines: 105 | if line.strip().startswith(ifconfig): 106 | break 107 | else: 108 | if not yesno("\nDidn't find an '%s' setting in rc.conf. You sure that you want to continue?" % ifconfig): 109 | return 110 | 111 | print("\nThe generated rc_conf:\n%s" % rc_conf) 112 | print("bootstrap-bsd-url:", bu.bsd_url) 113 | if 'bootstrap-destroygeom' in env.instance.config: 114 | print("bootstrap-destroygeom:", env.instance.config['bootstrap-destroygeom']) 115 | if 'bootstrap-zfsinstall' in env.instance.config: 116 | print("bootstrap-zfsinstall:", env.instance.config['bootstrap-zfsinstall']) 117 | system_pool_name = env.instance.config.get('bootstrap-system-pool-name', 'system') 118 | print("bootstrap-system-pool-name:", system_pool_name) 119 | data_pool_name = env.instance.config.get('bootstrap-data-pool-name', 'tank') 120 | print("bootstrap-data-pool-name:", data_pool_name) 121 | swap_size = env.instance.config.get('bootstrap-swap-size', '%iM' % (realmem * 2)) 122 | print("bootstrap-swap-size:", swap_size) 123 | system_pool_size = env.instance.config.get('bootstrap-system-pool-size', '20G') 124 | print("bootstrap-system-pool-size:", system_pool_size) 125 | firstboot_update = value_asbool(env.instance.config.get('bootstrap-firstboot-update', 'false')) 126 | print("bootstrap-firstboot-update:", firstboot_update) 127 | autoboot_delay = env.instance.config.get('bootstrap-autoboot-delay', '-1') 128 | print("bootstrap-autoboot-delay:", autoboot_delay) 129 | bootstrap_reboot = value_asbool(env.instance.config.get('bootstrap-reboot', 'true')) 130 | print("bootstrap-reboot:", bootstrap_reboot) 131 | bootstrap_packages = env.instance.config.get('bootstrap-packages', 'python27').split() 132 | if firstboot_update: 133 | bootstrap_packages.append('firstboot-freebsd-update') 134 | print("bootstrap-packages:", ", ".join(bootstrap_packages)) 135 | 136 | yes = env.instance.config.get('bootstrap-yes', False) 137 | if not (yes or yesno("\nContinuing will destroy the existing data on the following devices:\n %s\n\nContinue?" % ' '.join(bu.devices))): 138 | return 139 | 140 | # install FreeBSD in ZFS root 141 | devices_args = ' '.join('-d %s' % x for x in bu.devices) 142 | swap_arg = '' 143 | if swap_size: 144 | swap_arg = '-s %s' % swap_size 145 | system_pool_arg = '' 146 | if system_pool_size: 147 | system_pool_arg = '-z %s' % system_pool_size 148 | run('{destroygeom} {devices_args} -p {system_pool_name} -p {data_pool_name}'.format( 149 | destroygeom=bu.destroygeom, 150 | devices_args=devices_args, 151 | system_pool_name=system_pool_name, 152 | data_pool_name=data_pool_name)) 153 | run('{env_vars}{zfsinstall} {devices_args} -p {system_pool_name} -V 28 -u {bsd_url} {swap_arg} {system_pool_arg}'.format( 154 | env_vars=bu.env_vars, 155 | zfsinstall=bu.zfsinstall, 156 | devices_args=devices_args, 157 | system_pool_name=system_pool_name, 158 | bsd_url=bu.bsd_url, 159 | swap_arg=swap_arg, 160 | system_pool_arg=system_pool_arg), shell=False) 161 | # create partitions for data pool, but only if the system pool doesn't use 162 | # the whole disk anyway 163 | if system_pool_arg: 164 | for device in bu.devices: 165 | run('gpart add -t freebsd-zfs -l {data_pool_name}_{device} {device}'.format( 166 | data_pool_name=data_pool_name, 167 | device=device)) 168 | # mount devfs inside the new system 169 | if 'devfs on /mnt/dev' not in bu.mounts: 170 | run('mount -t devfs devfs /mnt/dev') 171 | # setup bare essentials 172 | run('cp /etc/resolv.conf /mnt/etc/resolv.conf', warn_only=True) 173 | bu.create_bootstrap_directories() 174 | bu.upload_bootstrap_files(template_context) 175 | 176 | if firstboot_update: 177 | run('''touch /mnt/firstboot''') 178 | run('''sysrc -f /mnt/etc/rc.conf firstboot_freebsd_update_enable=YES''') 179 | 180 | # update from config 181 | bootstrap_packages += env.instance.config.get('bootstrap-packages', '').split() 182 | # we need to install python here, because there is no way to install it via 183 | # ansible playbooks 184 | bu.install_pkg('/mnt', chroot=True, packages=bootstrap_packages) 185 | # set autoboot delay 186 | run('echo autoboot_delay=%s >> /mnt/boot/loader.conf' % autoboot_delay) 187 | bu.generate_remote_ssh_keys() 188 | # reboot 189 | if bootstrap_reboot: 190 | with settings(hide('warnings'), warn_only=True): 191 | run('reboot') 192 | 193 | 194 | @task 195 | def bootstrap(**kwargs): 196 | """ bootstrap an instance booted into mfsbsd (http://mfsbsd.vx.sk) 197 | """ 198 | with _mfsbsd(env, kwargs): 199 | _bootstrap() 200 | 201 | 202 | @task 203 | def fetch_assets(**kwargs): 204 | """ download bootstrap assets to control host. 205 | If present on the control host they will be uploaded to the target host during bootstrapping. 206 | """ 207 | # allow overwrites from the commandline 208 | env.instance.config.update(kwargs) 209 | BootstrapUtils().fetch_assets() 210 | --------------------------------------------------------------------------------