├── vbox2 ├── cloud │ └── .gitkeep ├── .env ├── ignition │ └── ssh.yaml ├── profiles │ └── pxe.json ├── groups │ ├── default.json │ └── pxe │ │ └── default.json └── README.md ├── vbox ├── group_vars │ ├── all │ └── local │ │ ├── plaintextvars.yaml │ │ ├── coreos.yaml │ │ └── vault.yaml ├── cbm │ ├── dnsmasq.data │ │ ├── dnsmasq.leases │ │ └── dnsmasq.conf │ ├── dnsmasq │ ├── bootcfg │ └── data │ │ ├── specs │ │ ├── normal │ │ │ └── spec.json │ │ ├── default │ │ │ └── spec.json │ │ └── initialize │ │ │ └── spec.json │ │ ├── ignition │ │ ├── default.json │ │ └── initialize.json │ │ └── config.yaml ├── ansible.cfg ├── hosts └── public-files │ └── vbox-key.pub ├── roles ├── cbm │ ├── tests │ │ ├── inventory │ │ └── test.yml │ ├── defaults │ │ └── main.yml │ ├── handlers │ │ └── main.yml │ ├── templates │ │ ├── default.j2 │ │ ├── spec-normal.json │ │ ├── spec-default.json │ │ ├── default.json │ │ ├── spec-initialize.json │ │ └── dnsmasq.conf │ ├── files │ │ ├── set-hostname-default.service │ │ ├── verify-coreos-sig.sh │ │ └── config.yaml │ ├── vars │ │ └── main.yml │ ├── .travis.yml │ ├── tasks │ │ ├── main.yml │ │ └── configure.yaml │ ├── README.md │ └── meta │ │ └── main.yml ├── cbm-machine │ ├── templates │ │ ├── initialize.j2 │ │ └── initialize.json │ ├── files │ │ ├── set-hostname.service │ │ └── ansible-setup.sh │ ├── vars │ │ └── main.yaml │ └── tasks │ │ └── main.yaml └── common │ └── tasks │ └── add-to-git.yaml ├── .gitignore ├── austin ├── group_vars │ └── local │ │ ├── vars.yaml │ │ └── vault.yaml ├── ansible.cfg ├── hosts └── public-files │ ├── austin-key.pub │ └── austin-cluster-key.pub ├── undionly.kpxe ├── cbm-machines.yaml ├── hosts ├── ansible ├── hosts └── coreos-setup.markdown ├── make-cbm.yaml ├── powerdown-utility.yaml ├── powerup-utility.yaml ├── decrypt-secret-files.yaml ├── docs ├── virtualbox-ipxe.markdown ├── sharing-secrets.markdown ├── initial-setup.markdown ├── manual-setup.markdown └── utility-machine.markdown ├── encrypt-secret-files.yaml ├── make-environment.yaml ├── dnsmasq.conf ├── README.md └── LICENSE /vbox2/cloud/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /vbox/group_vars/all: -------------------------------------------------------------------------------- 1 | env: vbox -------------------------------------------------------------------------------- /roles/cbm/tests/inventory: -------------------------------------------------------------------------------- 1 | localhost -------------------------------------------------------------------------------- /vbox/cbm/dnsmasq.data/dnsmasq.leases: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /vbox/cbm/dnsmasq: -------------------------------------------------------------------------------- 1 | /usr/local/sbin/dnsmasq -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | secret-files/ 3 | vault-password 4 | assets/ 5 | -------------------------------------------------------------------------------- /austin/group_vars/local/vars.yaml: -------------------------------------------------------------------------------- 1 | plaintext_files: "{{ secret_files }}" -------------------------------------------------------------------------------- /roles/cbm/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # defaults file for coreos-baremetal 3 | -------------------------------------------------------------------------------- /roles/cbm/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # handlers file for coreos-baremetal 3 | -------------------------------------------------------------------------------- /vbox/group_vars/local/plaintextvars.yaml: -------------------------------------------------------------------------------- 1 | plaintext_files: "{{ secret_files }}" -------------------------------------------------------------------------------- /vbox/cbm/bootcfg: -------------------------------------------------------------------------------- 1 | /Users/spencer/gowork/src/github.com/coreos/coreos-baremetal/bin/bootcfg -------------------------------------------------------------------------------- /austin/ansible.cfg: -------------------------------------------------------------------------------- 1 | [defaults] 2 | inventory = hosts 3 | vault_password_file = vault-password -------------------------------------------------------------------------------- /undionly.kpxe: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SpencerBrown/coreos-pxe-install/HEAD/undionly.kpxe -------------------------------------------------------------------------------- /roles/cbm/tests/test.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - hosts: localhost 3 | remote_user: root 4 | roles: 5 | - coreos-baremetal -------------------------------------------------------------------------------- /roles/cbm/templates/default.j2: -------------------------------------------------------------------------------- 1 | {% do cbm_ignition.update({'passwd':cbm_sshkey}) %} 2 | 3 | {{ cbm_ignition | to_nice_json }} -------------------------------------------------------------------------------- /vbox/ansible.cfg: -------------------------------------------------------------------------------- 1 | [defaults] 2 | inventory = hosts 3 | vault_password_file = vault-password 4 | jinja2_extensions = jinja2.ext.do -------------------------------------------------------------------------------- /vbox2/.env: -------------------------------------------------------------------------------- 1 | export BOOTCFG_DATA_PATH=/Users/spencer/.bootcfg 2 | export BOOTCFG_ASSETS_PATH=/Users/spencer/.bootcfg/assets 3 | -------------------------------------------------------------------------------- /cbm-machines.yaml: -------------------------------------------------------------------------------- 1 | - name: Set up new CoreOS machine 2 | hosts: coreos 3 | gather_facts: no 4 | roles: 5 | - cbm-machine 6 | -------------------------------------------------------------------------------- /hosts: -------------------------------------------------------------------------------- 1 | # Default inventory file for playbooks not specific to an environment. 2 | 3 | [local] 4 | localhost ansible_connection=local -------------------------------------------------------------------------------- /roles/cbm-machine/templates/initialize.j2: -------------------------------------------------------------------------------- 1 | {% do cbm_ignition.update({'passwd':cbm_sshkey}) %} 2 | 3 | {{ cbm_ignition | to_nice_json }} -------------------------------------------------------------------------------- /ansible/hosts: -------------------------------------------------------------------------------- 1 | [coreos] 2 | 10.2.0.197 3 | 4 | [coreos:vars] 5 | ansible_ssh_user=core 6 | ansible_python_interpreter="PATH=/home/core:$PATH python" 7 | -------------------------------------------------------------------------------- /make-cbm.yaml: -------------------------------------------------------------------------------- 1 | - name: Set up the CoreOS Bare Metal configuration for an environment 2 | hosts: local 3 | gather_facts: no 4 | roles: 5 | - cbm 6 | -------------------------------------------------------------------------------- /roles/common/tasks/add-to-git.yaml: -------------------------------------------------------------------------------- 1 | - name: Add files to git 2 | command: "git add {{ inventory_dir }}" 3 | args: 4 | warn: no 5 | changed_when: false -------------------------------------------------------------------------------- /roles/cbm/files/set-hostname-default.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | [Service] 3 | Type=oneshot 4 | ExecStart=/usr/bin/hostnamectl set-hostname default 5 | [Install] 6 | WantedBy=multi-user.target -------------------------------------------------------------------------------- /roles/cbm-machine/files/set-hostname.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | [Service] 3 | Type=oneshot 4 | ExecStart=/usr/bin/hostnamectl set-hostname $HOSTNAME 5 | [Install] 6 | WantedBy=multi-user.target -------------------------------------------------------------------------------- /vbox/group_vars/local/coreos.yaml: -------------------------------------------------------------------------------- 1 | boot_server_ip: "10.3.0.152" 2 | boot_server_ip_base: "10.3.0.0" 3 | boot_server_ip_netmask: "255.255.252.0" 4 | coreos_channel: alpha 5 | coreos_release: "current" -------------------------------------------------------------------------------- /powerdown-utility.yaml: -------------------------------------------------------------------------------- 1 | - name: power down the Arch Linux utility machine 2 | hosts: utility 3 | sudo: yes 4 | tasks: 5 | 6 | - name: shut it down 7 | command: "systemctl poweroff" 8 | ignore_errors: yes -------------------------------------------------------------------------------- /powerup-utility.yaml: -------------------------------------------------------------------------------- 1 | - name: Wake up the Arch Linux utility machine 2 | hosts: utility 3 | gather_facts: no 4 | tasks: 5 | 6 | - name: wake it up (OS X only, must "brew install wakeonlan" 7 | local_action: "command wakeonlan {{ mac_address }}" -------------------------------------------------------------------------------- /roles/cbm/vars/main.yml: -------------------------------------------------------------------------------- 1 | 2 | cbm_gopath: "{{ lookup('env', 'GOPATH') }}" 3 | cbm_ignition: "{{ { 'ignitionVersion': 1 } }}" 4 | cbm_sshkey: "{{ { 'users': [ { 'name': 'core', 'sshAuthorizedKeys': [ lookup('file', env+'/public-files/'+env+'-key.pub') ] } ] } }}" 5 | -------------------------------------------------------------------------------- /vbox/hosts: -------------------------------------------------------------------------------- 1 | [local] 2 | localhost ansible_connection=local 3 | 4 | [coreos] 5 | vbox-01 ansible_host="10.3.0.210" boot_mac="08:00:27:ba:be:10" boot_disk="sda" boot_state=initialize 6 | 7 | [coreos:vars] 8 | ansible_user=core 9 | ansible_python_interpreter="PATH=/home/core:$PATH python" -------------------------------------------------------------------------------- /vbox2/ignition/ssh.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | ignition_version: 1 3 | {{ if index . "ssh_authorized_keys" }} 4 | passwd: 5 | users: 6 | - name: core 7 | ssh_authorized_keys: 8 | {{ range $element := .ssh_authorized_keys }} 9 | - {{$element}} 10 | {{end}} 11 | {{end}} 12 | -------------------------------------------------------------------------------- /roles/cbm-machine/vars/main.yaml: -------------------------------------------------------------------------------- 1 | 2 | cbm_ignition: "{{ { 'ignitionVersion': 1 } }}" 3 | cbm_sshkey: "{{ { 'users': [ { 'name': 'core', 'sshAuthorizedKeys': [ lookup('file', env+'/public-files/'+env+'-key.pub') ] } ] } }}" 4 | cbm_disk: "{{ { 'disks': [ { 'device': '/dev/'boot_disk, 'wipeTable': true, 'partitions': [ { 'label': 'ROOT', 'number': 0 } ] } ] }}" -------------------------------------------------------------------------------- /vbox/cbm/data/specs/normal/spec.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "normal", 3 | "boot": { 4 | "kernel": "/assets/coreos/current/coreos_production_pxe.vmlinuz", 5 | "initrd": [ "/assets/coreos/current/coreos_production_pxe_image.cpio.gz" ], 6 | "cmdline": { 7 | "coreos.autologin": "", 8 | "root": "/dev/sda1" 9 | } 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /roles/cbm/files/verify-coreos-sig.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | gpg --import < "$1/CoreOS_Image_Signing_Key.asc" 4 | echo "Adding trust for CoreOS signing key:" 5 | echo "04127D0BFABEC8871FFB2CCE50E0885593D2DCB4:6:" | gpg --import-ownertrust 6 | gpg --verify "$1/coreos_production_pxe.vmlinuz.sig" 7 | gpg --verify "$1/coreos_production_pxe_image.cpio.gz.sig" 8 | -------------------------------------------------------------------------------- /roles/cbm/templates/spec-normal.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "normal", 3 | "boot": { 4 | "kernel": "/assets/coreos/{{ coreos_release }}/coreos_production_pxe.vmlinuz", 5 | "initrd": [ "/assets/coreos/{{ coreos_release }}/coreos_production_pxe_image.cpio.gz" ], 6 | "cmdline": { 7 | "coreos.autologin": "", 8 | "root": "/dev/sda1" 9 | } 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /austin/hosts: -------------------------------------------------------------------------------- 1 | [local] 2 | localhost ansible_connection=local env=austin 3 | 4 | [utility] 5 | #TODO: use fact caching to remember the mac address of a powered down utility machine 6 | utility ansible_ssh_host=10.2.0.202 ansible_ssh_user=admin mac_address="d0:50:99:88:e3:7d" 7 | 8 | [cluster] 9 | austin-1 ansible_ssh_host=10.0.0.1 ansible_ssh_user=core mac_address="90:2b:34:14:f6:6a" motherboard="Gigabyte H77-DS3H rev 1.0" -------------------------------------------------------------------------------- /vbox/public-files/vbox-key.pub: -------------------------------------------------------------------------------- 1 | ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC09TuJHaqfxZXeEcTUwdilIeqdeq5h+rkpTasno7CfdeKM1stFOGqFVjHiFixC+cOtZW+Sxa2aMNKEnbdJwptKYyRy5EKJBzGpLxjvQk9hSldPJ0M4NCGSMz7SgwgSw+UqdkVfhfz0Lbk0urtdUL8bs9ap5c2poUsWE5j8WS/MvCL18adPxDQqFgnpX1sF1HFZ7+sgykF3oG6rI4dsocTkDSIwvVAqpZY9FjqvHrjAfaTAKpQ4ypGZ/Th3zIvbzc7PpRf89u/hi+1hez3lfuKyukcTmbXnwzarhvlmj9xcxs2w7oN2p0D+bn1Zt6ZBv6nPa8J7T8ptdyzq1MJ/vKaL spencer@Spencers-MacBook-Pro-2.local 2 | -------------------------------------------------------------------------------- /austin/public-files/austin-key.pub: -------------------------------------------------------------------------------- 1 | ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCuZjSqj8pIoDgyGOHVarJkaKK2rOn4cD6dWl9FSAkHNEqJqBUnk4eOBJEs39YL2+eci3JKJZ6tpPQfm4z2TSwBxcTNVhrOCDfoa6Ai3YGinY9b9BytNcI13i93vbkieoR/NCyGfvcRaDmEUUvOwB7ffiB/8FOJJc1MrmtWS2Tq8Fc0ZU1mzY3d5VGq4CzRLHLS97djQ24QAhcak1fbPQQBPBDuHJYUstq5xKiaLbLC5/iFU6Z7umA2G2LEMcsfyc2za135ICeeaBiKAioTekV5DK3/7DbvQpmglZjZwP3DyFbFNvC9EeSK0W6PM34dMiPLPdN3TqUYXiVNGITL//3r spencer@Spencers-MacBook-Pro-2.local 2 | -------------------------------------------------------------------------------- /austin/public-files/austin-cluster-key.pub: -------------------------------------------------------------------------------- 1 | ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCuJSfbWKlpyOjNhBUiO13WY2JP8NIgA5FTtuVtB2HmFHqElQZnUM5xR+sR1NsJ5u46UbvIGTrrNrZopKsu0kBkFwOT5beW6OQCeStO9ujhF1Soowqvj8X5m5wo1xwz/sI9blFA5Og9tYqHR9QKW6xevvyLtc//2aEZhyUBJyehxoO3VNPdmoDdeWjdsciQgIHczwenPBD7gi5khChaJ1J6iLC66wyJwbvlJRVoo9739EPzIO6tlxXnwWtgfrNsBOh9h/zUXaWI8xazWM8IVgaufMAWrtfF2xXysd0H3/CfgmdlCD5AuOgotBseiXJS6b4FZ7duFGZrpQ7j8a/PNYPP spencer@Spencers-MacBook-Pro-2.local 2 | -------------------------------------------------------------------------------- /roles/cbm-machine/tasks/main.yaml: -------------------------------------------------------------------------------- 1 | 2 | - name: Set up machine for Ansible if in initialize state 3 | script: ansible-setup.sh 4 | when: boot_state == "initialize" 5 | 6 | - name: create initialize.json Ignition file if machine in initialize state 7 | # implies can only initialize one machine at a time 8 | template: 9 | src: initialize.j2 10 | dest: "cbm/data/ignition/initialize.json" 11 | delegate_to: localhost 12 | when: boot_state == "initialize" -------------------------------------------------------------------------------- /vbox/cbm/data/specs/default/spec.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "default", 3 | "boot": { 4 | "kernel": "/assets/coreos/current/coreos_production_pxe.vmlinuz", 5 | "initrd": [ "/assets/coreos/current/coreos_production_pxe_image.cpio.gz" ], 6 | "cmdline": { 7 | "coreos.config.url": "http://10.3.0.152:8080/ignition?uuid=dontmatch", 8 | "coreos.autologin": "", 9 | "coreos.first_boot": "1" 10 | } 11 | }, 12 | "ignition_id": "default.json" 13 | } 14 | -------------------------------------------------------------------------------- /vbox2/profiles/pxe.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "pxe", 3 | "name": "CoreOS with SSH", 4 | "ignition_id": "ssh.yaml", 5 | "boot": { 6 | "kernel": "/assets/coreos/1053.2.0/coreos_production_pxe.vmlinuz", 7 | "initrd": ["/assets/coreos/1053.2.0/coreos_production_pxe_image.cpio.gz"], 8 | "cmdline": { 9 | "coreos.autologin": "", 10 | "coreos.config.url": "http://bootcfg.foo:8080/ignition?uuid=${uuid}&mac=${net0/mac:hexhyp}", 11 | "coreos.first_boot": "" 12 | } 13 | } 14 | } -------------------------------------------------------------------------------- /roles/cbm/templates/spec-default.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "default", 3 | "boot": { 4 | "kernel": "/assets/coreos/{{ coreos_release }}/coreos_production_pxe.vmlinuz", 5 | "initrd": [ "/assets/coreos/{{ coreos_release }}/coreos_production_pxe_image.cpio.gz" ], 6 | "cmdline": { 7 | "coreos.config.url": "http://{{ boot_server_ip }}:8080/ignition?uuid=dontmatch", 8 | "coreos.autologin": "", 9 | "coreos.first_boot": "1" 10 | } 11 | }, 12 | "ignition_id": "default.json" 13 | } 14 | -------------------------------------------------------------------------------- /roles/cbm-machine/files/ansible-setup.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | if [ -e get-pip.py ] 4 | then 5 | echo "Already set up for Ansible" 6 | exit 7 | fi 8 | 9 | wget https://bitbucket.org/squeaky/portable-pypy/downloads/pypy-4.0.1-linux_x86_64-portable.tar.bz2 10 | tar -jxf pypy-4.0.1-linux_x86_64-portable.tar.bz2 11 | rm pypy-4.0.1-linux_x86_64-portable.tar.bz2 12 | mv pypy-4.0.1-linux_x86_64-portable pypy 13 | ln -s pypy/bin/pypy python 14 | curl -O https://bootstrap.pypa.io/get-pip.py 15 | # ./python get-pip.py -------------------------------------------------------------------------------- /vbox/cbm/data/specs/initialize/spec.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "initialize", 3 | "boot": { 4 | "kernel": "/assets/coreos/current/coreos_production_pxe.vmlinuz", 5 | "initrd": [ "/assets/coreos/current/coreos_production_pxe_image.cpio.gz" ], 6 | "cmdline": { 7 | "coreos.config.url": "http://10.3.0.152:8080/ignition?uuid=${uuid}&mac=${net0/mac:hexhyp}", 8 | "coreos.autologin": "", 9 | "coreos.first_boot": "1", 10 | "root": "/dev/sda1" 11 | } 12 | }, 13 | "ignition_id": "initialize.json" 14 | } 15 | -------------------------------------------------------------------------------- /roles/cbm/templates/default.json: -------------------------------------------------------------------------------- 1 | { 2 | "ignitionVersion": 1, 3 | "passwd": { 4 | "users": [ 5 | { 6 | "name": "core", 7 | "sshAuthorizedKeys": [ 8 | "{{ lookup('file', env+'/public-files/'+env+'-key.pub') }}" 9 | ] 10 | } 11 | ] 12 | }, 13 | "systemd": { 14 | "units": [ 15 | { 16 | "name": "set-hostname.service", 17 | "enable": true, 18 | "contents": "{{ lookup('file', 'set-hostname-default.service') | replace('\n', '\\n') }}" 19 | } 20 | ] 21 | } 22 | } -------------------------------------------------------------------------------- /roles/cbm/templates/spec-initialize.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "initialize", 3 | "boot": { 4 | "kernel": "/assets/coreos/{{ coreos_release }}/coreos_production_pxe.vmlinuz", 5 | "initrd": [ "/assets/coreos/{{ coreos_release }}/coreos_production_pxe_image.cpio.gz" ], 6 | "cmdline": { 7 | "coreos.config.url": "http://{{ boot_server_ip }}:8080/ignition?uuid=${uuid}&mac=${net0/mac:hexhyp}", 8 | "coreos.autologin": "", 9 | "coreos.first_boot": "1", 10 | "root": "/dev/sda1" 11 | } 12 | }, 13 | "ignition_id": "initialize.json" 14 | } 15 | -------------------------------------------------------------------------------- /vbox2/groups/default.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "default", 3 | "name": "PXE CoreOS alpha", 4 | "profile": "pxe", 5 | "metadata": { 6 | "ssh_authorized_keys": ["ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCtVlvW+aP795lOoSbE3QiNmrAgyQ8ZAlPS7wM4G0yw5s1SfwROHzmWziCt5vUgLQAXV2gugr+lVDb3NLjTeIz1wgWUCiawh91HBo5FYF0Lixo0/qo1mvaARt03u2dZipIzMeiLpwuhSerBQYsrNWOqb3O8JbrRskY1T6i1I6J03XoXtmJ2nHOX47Bqmx9/YhXVYLwWcrkLulH/c52/Kkl8X/5y17xbnTunS1C7BKU7j/tcOJtJsdncXvQVckP77+xBGsx68LgHCtrNhXIg8wyo/Y8hJdguZDX8BDWYJ2ZjI6j9xGB5DhkFbAxetIU9HxOdXIvIkE5beb9BsSyD52yt spencer@Spencers-MBP-2"] 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /vbox2/groups/pxe/default.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "default", 3 | "name": "PXE CoreOS alpha", 4 | "profile": "pxe", 5 | "metadata": { 6 | "ssh_authorized_keys": ["ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCtVlvW+aP795lOoSbE3QiNmrAgyQ8ZAlPS7wM4G0yw5s1SfwROHzmWziCt5vUgLQAXV2gugr+lVDb3NLjTeIz1wgWUCiawh91HBo5FYF0Lixo0/qo1mvaARt03u2dZipIzMeiLpwuhSerBQYsrNWOqb3O8JbrRskY1T6i1I6J03XoXtmJ2nHOX47Bqmx9/YhXVYLwWcrkLulH/c52/Kkl8X/5y17xbnTunS1C7BKU7j/tcOJtJsdncXvQVckP77+xBGsx68LgHCtrNhXIg8wyo/Y8hJdguZDX8BDWYJ2ZjI6j9xGB5DhkFbAxetIU9HxOdXIvIkE5beb9BsSyD52yt spencer@Spencers-MBP-2"] 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /roles/cbm/.travis.yml: -------------------------------------------------------------------------------- 1 | --- 2 | language: python 3 | python: "2.7" 4 | 5 | # Use the new container infrastructure 6 | sudo: false 7 | 8 | # Install ansible 9 | addons: 10 | apt: 11 | packages: 12 | - python-pip 13 | 14 | install: 15 | # Install ansible 16 | - pip install ansible 17 | 18 | # Check ansible version 19 | - ansible --version 20 | 21 | # Create ansible.cfg with correct roles_path 22 | - printf '[defaults]\nroles_path=../' >ansible.cfg 23 | 24 | script: 25 | # Basic role syntax check 26 | - ansible-playbook tests/test.yml -i tests/inventory --syntax-check 27 | 28 | notifications: 29 | webhooks: https://galaxy.ansible.com/api/v1/notifications/ -------------------------------------------------------------------------------- /vbox/cbm/data/ignition/default.json: -------------------------------------------------------------------------------- 1 | 2 | { 3 | "ignitionVersion": 1, 4 | "passwd": { 5 | "users": [ 6 | { 7 | "name": "core", 8 | "sshAuthorizedKeys": [ 9 | "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC09TuJHaqfxZXeEcTUwdilIeqdeq5h+rkpTasno7CfdeKM1stFOGqFVjHiFixC+cOtZW+Sxa2aMNKEnbdJwptKYyRy5EKJBzGpLxjvQk9hSldPJ0M4NCGSMz7SgwgSw+UqdkVfhfz0Lbk0urtdUL8bs9ap5c2poUsWE5j8WS/MvCL18adPxDQqFgnpX1sF1HFZ7+sgykF3oG6rI4dsocTkDSIwvVAqpZY9FjqvHrjAfaTAKpQ4ypGZ/Th3zIvbzc7PpRf89u/hi+1hez3lfuKyukcTmbXnwzarhvlmj9xcxs2w7oN2p0D+bn1Zt6ZBv6nPa8J7T8ptdyzq1MJ/vKaL spencer@Spencers-MacBook-Pro-2.local" 10 | ] 11 | } 12 | ] 13 | } 14 | } -------------------------------------------------------------------------------- /vbox/cbm/data/ignition/initialize.json: -------------------------------------------------------------------------------- 1 | 2 | { 3 | "ignitionVersion": 1, 4 | "passwd": { 5 | "users": [ 6 | { 7 | "name": "core", 8 | "sshAuthorizedKeys": [ 9 | "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC09TuJHaqfxZXeEcTUwdilIeqdeq5h+rkpTasno7CfdeKM1stFOGqFVjHiFixC+cOtZW+Sxa2aMNKEnbdJwptKYyRy5EKJBzGpLxjvQk9hSldPJ0M4NCGSMz7SgwgSw+UqdkVfhfz0Lbk0urtdUL8bs9ap5c2poUsWE5j8WS/MvCL18adPxDQqFgnpX1sF1HFZ7+sgykF3oG6rI4dsocTkDSIwvVAqpZY9FjqvHrjAfaTAKpQ4ypGZ/Th3zIvbzc7PpRf89u/hi+1hez3lfuKyukcTmbXnwzarhvlmj9xcxs2w7oN2p0D+bn1Zt6ZBv6nPa8J7T8ptdyzq1MJ/vKaL spencer@Spencers-MacBook-Pro-2.local" 10 | ] 11 | } 12 | ] 13 | } 14 | } -------------------------------------------------------------------------------- /decrypt-secret-files.yaml: -------------------------------------------------------------------------------- 1 | - name: Decrypt secret files in vault.yaml and place them in secret-files directory 2 | hosts: local 3 | gather_facts: no 4 | tasks: 5 | 6 | - name: Ensure secret-files directory exists 7 | file: 8 | path: "{{ inventory_dir }}/secret-files" 9 | state: directory 10 | 11 | - name: Decrypt files and put in secret-files directory 12 | # note we have to use base64 command because Ansible copy contents doesn't support binary or trailing newlines 13 | shell: "printf '{{ item.value }}' | base64 --decode > {{ inventory_dir }}/secret-files/{{ item.key }}" 14 | with_dict: "{{ plaintext_files }}" 15 | no_log: true 16 | 17 | # secret_files variable is a hash like this: 18 | 19 | # secret_files: 20 | # file_42: "---base64 encoded string---" 21 | # another_file: "..another base64 encoded string representing the file's contents..." 22 | -------------------------------------------------------------------------------- /vbox/cbm/data/config.yaml: -------------------------------------------------------------------------------- 1 | api_version: v1alpha1 2 | groups: 3 | 4 | # The default spec is used if no other spec matches the machine. 5 | # It simply boots the machine using RAM only, with the SSH key. 6 | # You can login to the machine and inspect or modify it. 7 | - name: default 8 | spec: default 9 | 10 | # The initialize spec wipes and repartitions and formats /dev/sda and /dev/sdb, 11 | # creates the CoreOS root filesystem on /dev/sda, 12 | # and uses Ignition to initialize required things. 13 | # We normally require uuid: nobody, so this spec won't get triggered. 14 | # Change the require: section and restart bootcfg to initialize a machine. 15 | - name: initialize 16 | spec: initialize 17 | require: 18 | uuid: c322f332-e05a-449f-8939-8d4f2f012312 19 | 20 | # The normal spec simply boots the system. It assumes everything is already configured. 21 | - name: normal 22 | spec: normal 23 | require: 24 | uuid: nobody 25 | 26 | -------------------------------------------------------------------------------- /roles/cbm/files/config.yaml: -------------------------------------------------------------------------------- 1 | api_version: v1alpha1 2 | groups: 3 | 4 | # The default spec is used if no other spec matches the machine. 5 | # It simply boots the machine using RAM only, with the SSH key. 6 | # You can login to the machine and inspect or modify it. 7 | - name: default 8 | spec: default 9 | 10 | # The initialize spec wipes and repartitions and formats /dev/sda and /dev/sdb, 11 | # creates the CoreOS root filesystem on /dev/sda, 12 | # and uses Ignition to initialize required things. 13 | # We normally require uuid: nobody, so this spec won't get triggered. 14 | # Change the require: section and restart bootcfg to initialize a machine. 15 | - name: initialize 16 | spec: initialize 17 | require: 18 | #mac: 90:2b:34:14:f6:6a 19 | uuid: nobody 20 | 21 | # The normal spec simply boots the system. It assumes everything is already configured. 22 | - name: normal 23 | spec: normal 24 | require: 25 | #mac: 90:2b:34:14:f6:6a 26 | uuid: nobody 27 | 28 | -------------------------------------------------------------------------------- /roles/cbm-machine/templates/initialize.json: -------------------------------------------------------------------------------- 1 | { 2 | "ignitionVersion": 1, 3 | "passwd": { 4 | "users": [ 5 | { 6 | "name": "core", 7 | "sshAuthorizedKeys": [ 8 | "{{ lookup('file', env+'/public-files/'+env+'-key.pub') }}" 9 | ] 10 | } 11 | ] 12 | }, 13 | "systemd": { 14 | "units": [ 15 | { 16 | "name": "set-hostname.service", 17 | "enable": true, 18 | "contents": "{{ lookup('file', 'set-hostname.service') | | replace ('$HOSTNAME', inventory_hostname) | replace('\n', '\\n') }}" 19 | } 20 | ] 21 | }, 22 | "storage": { 23 | "disks": [ 24 | { 25 | "device": "/dev/{{ boot_disk }}", 26 | "wipeTable": true, 27 | "partitions": [ 28 | { 29 | "label": "ROOT", 30 | "number": 0 31 | } 32 | ] 33 | } 34 | ], 35 | "filesystems": [ 36 | { 37 | "device": "/dev/{{ boot_disk }}1", 38 | "format": "ext4", 39 | "create": { 40 | "force": true, 41 | "options": [ 42 | "-i 8192" 43 | ] 44 | } 45 | } 46 | ] 47 | } 48 | } -------------------------------------------------------------------------------- /roles/cbm/tasks/main.yml: -------------------------------------------------------------------------------- 1 | 2 | - name: Create cbm directories 3 | file: 4 | path: "{{ item }}" 5 | state: directory 6 | with_items: 7 | - cbm/data/ignition 8 | - cbm/data/cloud 9 | - cbm/data/specs/default 10 | - cbm/data/specs/initialize 11 | - cbm/data/specs/normal 12 | - cbm/assets/coreos 13 | - cbm/assets/pxe 14 | 15 | - name: Create symlinks to executables 16 | file: 17 | path: cbm/bootcfg 18 | src: "{{ cbm_gopath }}/src/github.com/coreos/coreos-baremetal/bin/bootcfg" 19 | state: link 20 | - file: 21 | path: cbm/dnsmasq 22 | src: /usr/local/sbin/dnsmasq 23 | state: link 24 | 25 | - name: create variables file for boot_server_ip, boot_server_ip_base, boot_server_ip_netmask, coreos_release, and coreos_channel 26 | copy: 27 | dest: group_vars/local/coreos.yaml 28 | content: "boot_server_ip: \"10.2.0.200\"\nboot_server_ip_base: \"10.2.0.0\"\nboot_server_ip_netmask: \"255.255.255.0\"\ncoreos_channel: alpha\ncoreos_release: \"current\"" 29 | force: no 30 | 31 | - name: Download undionly.kpxe if needed 32 | stat: 33 | path: cbm/assets/pxe/undionly.kpxe.0 34 | register: undionly_file 35 | - get_url: 36 | dest: cbm/assets/pxe/undionly.kpxe.0 37 | url: http://boot.ipxe.org/undionly.kpxe 38 | when: undionly_file.stat.exists==False 39 | 40 | - include: configure.yaml -------------------------------------------------------------------------------- /roles/cbm/README.md: -------------------------------------------------------------------------------- 1 | Role Name 2 | ========= 3 | 4 | A brief description of the role goes here. 5 | 6 | Requirements 7 | ------------ 8 | 9 | Any pre-requisites that may not be covered by Ansible itself or the role should be mentioned here. For instance, if the role uses the EC2 module, it may be a good idea to mention in this section that the boto package is required. 10 | 11 | Role Variables 12 | -------------- 13 | 14 | A description of the settable variables for this role should go here, including any variables that are in defaults/main.yml, vars/main.yml, and any variables that can/should be set via parameters to the role. Any variables that are read from other roles and/or the global scope (ie. hostvars, group vars, etc.) should be mentioned here as well. 15 | 16 | Dependencies 17 | ------------ 18 | 19 | A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles. 20 | 21 | Example Playbook 22 | ---------------- 23 | 24 | Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too: 25 | 26 | - hosts: servers 27 | roles: 28 | - { role: username.rolename, x: 42 } 29 | 30 | License 31 | ------- 32 | 33 | BSD 34 | 35 | Author Information 36 | ------------------ 37 | 38 | An optional section for the role authors to include contact information, or a website (HTML is not allowed). 39 | -------------------------------------------------------------------------------- /ansible/coreos-setup.markdown: -------------------------------------------------------------------------------- 1 | # Setting up CoreOS for Ansible 2 | 3 | Almost all Ansible operations on remote servers require Python 2. 4 | CoreOS does not have Python, nor does it have a package manager to install Python. 5 | So we need to install Python by direct means. 6 | 7 | [An earlier project](https://github.com/defunctzombie/ansible-coreos-bootstrap) recommended using [PyPy](http://pypy.org) Python due to its lightweight nature and small set of dependencies on the operating system. 8 | 9 | This document shows a simplified method for installing PyPy on CoreOS, suitable for Ansible work. 10 | 11 | ## Download and install PyPy Portable version on CoreOS 12 | 13 | Login to the CoreOS machine as user `core`, and: 14 | 15 | ```bash 16 | wget https://bitbucket.org/squeaky/portable-pypy/downloads/pypy-4.0.1-linux_x86_64-portable.tar.bz2 17 | tar -jxf pypy-4.0.1-linux_x86_64-portable.tar.bz2 18 | rm pypy-4.0.1-linux_x86_64-portable.tar.bz2 19 | mv pypy-4.0.1-linux_x86_64-portable pypy 20 | ln -s pypy/bin/pypy python 21 | curl -O https://bootstrap.pypa.io/get-pip.py 22 | ./python get-pip.py 23 | ``` 24 | ## Specify the Python interpreter for Ansible 25 | 26 | Something like this in your inventory file: 27 | 28 | ``` 29 | [coreos] 30 | 10.2.0.197 31 | 32 | [coreos:vars] 33 | ansible_ssh_user=core 34 | ansible_python_interpreter=/home/core/python 35 | ``` 36 | 37 | Put the machines running CoreOS in the `coreos` group. They will all have the variables in the `[coreos:vars]` section. 38 | Ansible will then ssh to the machine as user `core`, and will find the PyPy version of Python in the proper place. -------------------------------------------------------------------------------- /docs/virtualbox-ipxe.markdown: -------------------------------------------------------------------------------- 1 | # Setting up VirtualBox iPXE support 2 | 3 | VirtualBox comes with virtual LAN Boot ROM that is a build of iPXE, with support for HTTP, but not for bzImage. 4 | bxImage is required for CoreOS, because they ship their PXE boot initramfs in cpio.bz format. 5 | 6 | So we have to build our own iPXE boot ROM and tell VirtualBox to do it. 7 | 8 | ## Setup 9 | 10 | [See here](http://ipxe.org/download) [and here](https://git.ipxe.org/ipxe.git/blob/HEAD:/src/config/vbox/README) for source material 11 | 12 | In short, get a Linux box, ensure you have: 13 | 14 | * git 15 | * gcc (version 3 or later) 16 | * binutils (version 2.18 or later) 17 | * make 18 | * perl 19 | * syslinux (for isolinux, only needed for building .iso images) 20 | * liblzma or xz header files 21 | * zlib, binutils and libiberty header files (only needed for EFI builds) 22 | 23 | ## Build 24 | 25 | ```bash 26 | git clone git://git.ipxe.org/ipxe.git 27 | cd ipxe/src 28 | ``` 29 | 30 | Edit `config/general.h` and uncomment this line: 31 | 32 | `//#define IMAGE_BZIMAGE /* Linux bzImage image support */` 33 | 34 | Then: 35 | 36 | ```bash 37 | make CONFIG=vbox bin/virtio-net.isarom 38 | ``` 39 | 40 | ## Install 41 | 42 | ```bash 43 | vboxmanage setextradata global VBoxInternal/Devices/pcbios/0/Config/LanBootRom /virtio-net.isarom 44 | ``` 45 | 46 | ## Run 47 | 48 | Define a VM whose NIC is of type `virtio-net` (other types won't work any more for PXE booting). 49 | Change its boot settings to allow network boot. 50 | Set up your PXE server appropriately, and start the machine. -------------------------------------------------------------------------------- /roles/cbm/templates/dnsmasq.conf: -------------------------------------------------------------------------------- 1 | # Disable DNS service 2 | port=0 3 | 4 | # Set the username that dnsmasq will switch to after startup 5 | # note that dnsmasq must be started as root 6 | user=spencer 7 | 8 | # with no interface=, dnsmasq on OS X binds to UDP ports as follows: 9 | # 67 (DHCP) on * 10 | # 69 (TFTP) on adapter's IP e.g. 10.2.0.200 and also 127.0.0.1 11 | # 4011 (PXE proxyDHCP) on * 12 | # also a bunch of IPv6 listent 13 | #interface=en0 14 | 15 | # with listen-address, dnsmasq on OS X binds to UDP ports as follows: 16 | listen-address={{ boot_server_ip }} 17 | # 67 (DHCP) on * 18 | # 69 (TFTP) on listen-address IP 19 | # 4011 (PXE proxyDHCP) on * 20 | 21 | # bind-interfaces disallows binding to 0.0.0.0, because dnsmasq doesn't support that on OS X 22 | # this avoids a warning in the log 23 | bind-interfaces 24 | 25 | # Make DHCP run in proxy mode, it does not supply IP addresses, only PXE responses 26 | dhcp-range={{ boot_server_ip_base }},proxy,255.255.252.0 27 | 28 | # Relocate the dnsmasq.leases file which is created but not used 29 | dhcp-leasefile={{ inventory_dir }}/cbm/dnsmasq.data/dnsmasq.leases 30 | 31 | # Enable the TFTP server 32 | enable-tftp 33 | tftp-root={{ inventory_dir }}/cbm/assets/pxe 34 | 35 | # kill multicast for DHCP PXE 36 | dhcp-option=vendor:PXEClient,6,2b 37 | 38 | # set tag "ipxe" if request comes from iPXE ("iPXE" user class) 39 | dhcp-userclass=set:ipxe,iPXE 40 | 41 | # if PXE request came from regular PXE firmware, serve iPXE firmware (via TFTP) 42 | dhcp-boot=tag:!ipxe,undionly.kpxe 43 | pxe-service=tag:!ipxe,x86PC,"PXE chainload to iPXE",undionly.kpxe 44 | 45 | # if PXE request came from iPXE, grab an iPXE boot script from the bootcfg server 46 | dhcp-boot=tag:ipxe,http://{{ boot_server_ip }}:8080/boot.ipxe 47 | pxe-service=tag:ipxe,x86PC,"Run iPXE boot",http://{{ boot_server_ip }}:8080/boot.ipxe -------------------------------------------------------------------------------- /vbox/cbm/dnsmasq.data/dnsmasq.conf: -------------------------------------------------------------------------------- 1 | # Disable DNS service 2 | port=0 3 | 4 | # Set the username that dnsmasq will switch to after startup 5 | # note that dnsmasq must be started as root 6 | user=spencer 7 | 8 | # with no interface=, dnsmasq on OS X binds to UDP ports as follows: 9 | # 67 (DHCP) on * 10 | # 69 (TFTP) on adapter's IP e.g. 10.2.0.200 and also 127.0.0.1 11 | # 4011 (PXE proxyDHCP) on * 12 | # also a bunch of IPv6 listent 13 | #interface=en0 14 | 15 | # with listen-address, dnsmasq on OS X binds to UDP ports as follows: 16 | listen-address=10.3.0.152 17 | # 67 (DHCP) on * 18 | # 69 (TFTP) on listen-address IP 19 | # 4011 (PXE proxyDHCP) on * 20 | 21 | # bind-interfaces disallows binding to 0.0.0.0, because dnsmasq doesn't support that on OS X 22 | # this avoids a warning in the log 23 | bind-interfaces 24 | 25 | # Make DHCP run in proxy mode, it does not supply IP addresses, only PXE responses 26 | dhcp-range=10.3.0.0,proxy,255.255.252.0 27 | 28 | # Relocate the dnsmasq.leases file which is created but not used 29 | dhcp-leasefile=/Users/spencer/src/coreos-pxe-install/vbox/cbm/dnsmasq.data/dnsmasq.leases 30 | 31 | # Enable the TFTP server 32 | enable-tftp 33 | tftp-root=/Users/spencer/src/coreos-pxe-install/vbox/cbm/assets/pxe 34 | 35 | # kill multicast for DHCP PXE 36 | dhcp-option=vendor:PXEClient,6,2b 37 | 38 | # set tag "ipxe" if request comes from iPXE ("iPXE" user class) 39 | dhcp-userclass=set:ipxe,iPXE 40 | 41 | # if PXE request came from regular PXE firmware, serve iPXE firmware (via TFTP) 42 | dhcp-boot=tag:!ipxe,undionly.kpxe 43 | pxe-service=tag:!ipxe,x86PC,"PXE chainload to iPXE",undionly.kpxe 44 | 45 | # if PXE request came from iPXE, grab an iPXE boot script from the bootcfg server 46 | dhcp-boot=tag:ipxe,http://10.3.0.152:8080/boot.ipxe 47 | pxe-service=tag:ipxe,x86PC,"Run iPXE boot",http://10.3.0.152:8080/boot.ipxe -------------------------------------------------------------------------------- /encrypt-secret-files.yaml: -------------------------------------------------------------------------------- 1 | - name: Encrypt the secret files in the secret-files directory and store them in the file "vault" 2 | # each file in the secret-files directory will be converted to a base64 string, 3 | # then added to file "vault.yaml" as a first level variable whose name is "secret_". 4 | # Then we use ansible-vault to encrypt the file, move it to group_vars/local, and add it to git. 5 | 6 | hosts: local 7 | gather_facts: no 8 | tasks: 9 | 10 | - name: get rid of old vault.yaml if it's there 11 | file: 12 | path: "{{ inventory_dir }}/group_vars/local/vault.yaml" 13 | state: absent 14 | 15 | # vault.yaml creates variables that look like this: 16 | 17 | # 18 | # hush_var_1: "a secret string" 19 | # my_secrets_xxx: "another secret string" 20 | # secret_files: 21 | # file_42: "---base64 encoded string---" 22 | # another_file: "..another base64 encoded string representing the file's contents..." 23 | # ... 24 | 25 | - name: create secrets file "vault.yaml" 26 | file: 27 | path: "{{ inventory_dir }}/secret-files/temp" 28 | state: directory 29 | 30 | - copy: 31 | dest: "{{ inventory_dir }}/secret-files/temp/vault.yaml" 32 | content: "secret_files:\n" 33 | 34 | - name: add entries to vault.yaml representing the secret files 35 | # note: have to use linux/osx base64 command because Ansible lookup('file') does not preserve final newline character 36 | shell: "echo \" {{ item | basename }}: $(base64 < {{ item }})\" >> {{ inventory_dir }}/secret-files/temp/vault.yaml" 37 | with_fileglob: 38 | - "{{ inventory_dir }}/secret-files/*" 39 | 40 | - name: Encrypt the vault.yaml file, move it to group_vars 41 | command: "{{ item }}" 42 | with_items: 43 | - "ansible-vault encrypt --vault-password-file={{ inventory_dir }}/vault-password {{ inventory_dir }}/secret-files/temp/vault.yaml" 44 | - "mv {{ inventory_dir }}/secret-files/temp/vault.yaml {{ inventory_dir }}/group_vars/local/vault.yaml" 45 | - "rm -r {{ inventory_dir }}/secret-files/temp" 46 | 47 | - include: roles/common/tasks/add-to-git.yaml 48 | 49 | -------------------------------------------------------------------------------- /roles/cbm/tasks/configure.yaml: -------------------------------------------------------------------------------- 1 | 2 | - name: Create dnsmasq.conf and dnsmasq.leases 3 | template: 4 | src: dnsmasq.conf 5 | dest: cbm/dnsmasq.data/dnsmasq.conf 6 | - copy: 7 | dest: cbm/dnsmasq.data/dnsmasq.leases 8 | content: "" 9 | force: no 10 | 11 | - name: Create config.yaml 12 | copy: 13 | src: config.yaml 14 | dest: cbm/data/config.yaml 15 | force: no 16 | 17 | - name: create spec.json files for default, initialize, and normal 18 | template: 19 | src: "spec-{{ item }}.json" 20 | dest: "cbm/data/specs/{{ item }}/spec.json" 21 | with_items: 22 | - default 23 | - initialize 24 | - normal 25 | 26 | - name: create default.json Ignition file 27 | template: 28 | #src: default.json 29 | src: default.j2 30 | dest: "cbm/data/ignition/default.json" 31 | 32 | - name: If needed, download CoreOS pxe binaries and verify signatures 33 | stat: 34 | path: cbm/assets/coreos/{{ coreos_release }} 35 | register: coreos_release_directory 36 | - file: 37 | path: cbm/assets/coreos/{{ coreos_release }} 38 | state: directory 39 | when: coreos_release_directory.stat.exists==false 40 | - get_url: 41 | dest: "cbm/assets/coreos/{{ coreos_release }}/" 42 | url: "http://{{ coreos_channel }}.release.core-os.net/amd64-usr/{{ coreos_release }}/{{ item }}" 43 | with_items: 44 | - coreos_production_pxe.vmlinuz 45 | - coreos_production_pxe.vmlinuz.sig 46 | - coreos_production_pxe_image.cpio.gz 47 | - coreos_production_pxe_image.cpio.gz.sig 48 | when: coreos_release_directory.stat.exists==false 49 | - get_url: 50 | dest: "cbm/assets/coreos/{{ coreos_release }}/" 51 | url: "https://coreos.com/security/image-signing-key/CoreOS_Image_Signing_Key.asc" 52 | when: coreos_release_directory.stat.exists==false 53 | - script: "verify-coreos-sig.sh {{ inventory_dir }}/cbm/assets/coreos/{{ coreos_release }}" 54 | when: coreos_release_directory.stat.exists==false 55 | register: gpg_output 56 | #- debug: var=gpg_output 57 | - fail: 58 | msg: "CoreOS signature verification failed! BEWARE!" 59 | when: (coreos_release_directory.stat.exists==false) and (gpg_output.stderr | search('BAD')) 60 | 61 | -------------------------------------------------------------------------------- /make-environment.yaml: -------------------------------------------------------------------------------- 1 | - name: Create a new environment (top level directory name) 2 | hosts: local 3 | gather_facts: no 4 | tasks: 5 | 6 | - fail: 7 | msg: "Please specify the environment name (use the env variable)" 8 | when: env is not defined 9 | 10 | - name: Create directory structure 11 | file: 12 | path: "{{ item }}" 13 | state: directory 14 | with_items: 15 | - "{{ env }}/group_vars/local" 16 | - "{{ env }}/public-files" 17 | - "{{ env }}/secret-files" 18 | 19 | - name: Create hosts file 20 | copy: 21 | dest: "{{ env }}/hosts" 22 | content: "[local]\nlocalhost ansible_connection=local env={{ env }}" 23 | 24 | - name: Create ansible.cfg file 25 | copy: 26 | dest: "{{ env }}/ansible.cfg" 27 | content: "[defaults]\ninventory = hosts\nvault_password_file = vault-password" 28 | 29 | - name: create variable names file "plaintextvars.yaml" in group_vars defining "plaintext_files" variable 30 | # This will also ensure there's a file in group_vars/local, so that Git will commit the directories 31 | # content is plaintext_files: "{{ secret_files }}" 32 | copy: 33 | dest: "{{ env }}/group_vars/local/plaintextvars.yaml" 34 | content: "plaintext_files: {{ '\"{{' }} secret_files {{ '}}\"' }}" 35 | 36 | - name: create SSH keypair for this environment's machines 37 | command: "ssh-keygen -f ~/.ssh/{{ env }}-key -N \"\"" 38 | args: 39 | creates: "~/.ssh/{{ env }}-key" 40 | 41 | - name: copy private keys to secret-files directory 42 | copy: 43 | src: "{{ item }}" 44 | dest: "{{ env }}/secret-files/" 45 | mode: 0600 46 | with_items: 47 | - "~/.ssh/{{ env }}-key" 48 | 49 | - name: copy public keys to public-files directory 50 | copy: 51 | src: "{{ item }}" 52 | dest: "{{ env }}/public-files/" 53 | with_items: 54 | - "~/.ssh/{{ env }}-key.pub" 55 | 56 | - name: Create vault-password file with random password 57 | command: "true {{ lookup('password', env + '/vault-password chars=ascii_letters') }}" 58 | args: 59 | creates: "{{ env }}/vault-password" 60 | 61 | - include: roles/common/tasks/add-to-git.yaml 62 | -------------------------------------------------------------------------------- /docs/sharing-secrets.markdown: -------------------------------------------------------------------------------- 1 | # Secure sharing of secrets 2 | 3 | Each environment directory has a subdirectory `secret-files`, to hold files that have secret information. 4 | For example, you can place SSH private keys, SSL private keys, and Docker credentials in the `secret-files` directory. 5 | The `secret-files` directory is mentioned in the `.gitignore` file, so your secret files will never be included in a git commit or pushed to a remote git repository such as GitHub. 6 | 7 | So, how do you share secret files among team members? If you create new secret files, and you want another team member to work with them, 8 | you have to make them available by some means outside of the Git repository. This interrupts your Git workflow and adds a path for mistakes. 9 | 10 | This repository supports encrypting your secret files, committing and pushing the encrypted files via Git, and decrypting them back into their original form. 11 | The only secret that needs to be shared is a single "vault password" per environment. 12 | 13 | To use this feature, the original creator of the environment follows these steps: 14 | 15 | 1. Create your environment as described in the README. This will automatically add a new SSH keypair to your environment, and create a random encryption password saved in `vault-password`. 16 | 2. Add additional secret files to the `secret-files` directory, as appropriate. These can be binary or text, but should be relatively small files. 17 | 3. Run `ansible-playbook ../encrypt-secret-files.yaml`. 18 | 4. Commit the changes using Git, and push the changes to a shared repository such as Github. 19 | 20 | The team member who wishes to use the environment follows these steps: 21 | 22 | 1. Ask the creator for the environment's `vault-password` file contents. It will be a random string of 20 or so letters. Create your own `vault-password` file in the environment directory. 23 | 2. Clone and/or pull the shared repository locally, so that it's up-to-date. 24 | 3. `cd ` where `` is the name of your environment. 25 | 4. `ansible-playbook ../decrypt-secret-files.yaml`. 26 | 5. You may wish to copy the SSH keys for your environment to your `~/.ssh` directory. 27 | 28 | If you need to add or change secret files, make your changes in your `secret-files` directory, then run the `encrypt-secret-files.yaml` playbook and commit/push. 29 | 30 | Then, other team members can pull your changes and run the `decrypt-secret-files` playbook to update their local secret files. -------------------------------------------------------------------------------- /vbox2/README.md: -------------------------------------------------------------------------------- 1 | 2 | # Examples 3 | 4 | These examples network boot and provision VMs into CoreOS clusters using `bootcfg`. 5 | 6 | | Name | Description | CoreOS Version | FS | Docs | 7 | |------------|-------------|----------------|----|-----------| 8 | | pxe | CoreOS via iPXE | alpha/1053.2.0 | RAM | [reference](https://coreos.com/os/docs/latest/booting-with-ipxe.html) | 9 | | grub | CoreOS via GRUB2 Netboot | alpha/1053.2.0 | RAM | NA | 10 | | pxe-disk | CoreOS via iPXE, with a root filesystem | alpha/1053.2.0 | Disk | [reference](https://coreos.com/os/docs/latest/booting-with-ipxe.html) | 11 | | etcd, etcd-docker | iPXE boot a 3 node etcd cluster and proxy | alpha/1053.2.0 | RAM | [reference](https://coreos.com/os/docs/latest/cluster-architectures.html) | 12 | | etcd-install | Install a 3-node etcd cluster to disk | alpha/1053.2.0 | Disk | [reference](https://coreos.com/os/docs/latest/installing-to-disk.html) | 13 | | k8s, k8s-docker | Kubernetes cluster with 1 master and 2 workers, TLS-authentication | alpha/1053.2.0 | Disk | [tutorial](../Documentation/kubernetes.md) | 14 | | k8s-install | Install a Kubernetes cluster to disk (1 master) | alpha/1053.2.0 | Disk | [tutorial](../Documentation/kubernetes.md) | 15 | | bootkube | iPXE boot a self-hosted Kubernetes cluster (with bootkube) | alpha/1053.2.0 | Disk | [tutorial](../Documentation/bootkube.md) | 16 | | bootkube-install | Install a self-hosted Kubernetes cluster (with bootkube) | alpha/1053.2.0 | Disk | [tutorial](../Documentation/bootkube.md) | 17 | 18 | ## Tutorials 19 | 20 | Get started running `bootcfg` on your Linux machine to network boot and provision clusters of VMs or physical hardware. 21 | 22 | * [bootcfg with rkt](../Documentation/getting-started-rkt.md) 23 | * [bootcfg with Docker](../Documentation/getting-started-docker.md) 24 | * [Kubernetes v1.2.4](../Documentation/kubernetes.md) 25 | * [Self-hosted Kubernetes](../Documentation/bootkube.md) (experimental) 26 | 27 | ## Experimental 28 | 29 | These examples demonstrate booting and provisioning various (often experimental) CoreOS clusters. They have **NOT** been hardened for production yet. You should write or adapt Ignition configs to suit your needs and hardware. 30 | 31 | ## SSH Keys 32 | 33 | Most examples allow `ssh_authorized_keys` to be added for the `core` user as machine group metadata. 34 | 35 | # /var/lib/bootcfg/groups/default.json 36 | { 37 | "name": "Example Machine Group", 38 | "profile": "pxe", 39 | "metadata": { 40 | "ssh_authorized_keys": ["ssh-rsa pub-key-goes-here"] 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /dnsmasq.conf: -------------------------------------------------------------------------------- 1 | # Requires these variables: 2 | # boot_server_ip: IP of the boot server 3 | # boot_server_ip_base: IP base of the boot server's local network, this should cover the booted server's IP range e.g. 10.2.0.0 4 | # boot_server_netmask: netmask covering the booted servers' IP range e.g. 255.255.255.0 5 | # inventory_dir: base directory for bootcfg data 6 | 7 | # Disable DNS service 8 | port=0 9 | 10 | # Set the username that dnsmasq will switch to after startup 11 | # note that dnsmasq must be started as root 12 | user=spencer 13 | 14 | # with no interface=, dnsmasq on OS X binds to UDP ports as follows: 15 | # 67 (DHCP) on * 16 | # 69 (TFTP) on adapter's IP e.g. 10.2.0.200 and also 127.0.0.1 17 | # 4011 (PXE proxyDHCP) on * 18 | # also a bunch of IPv6 listens 19 | #interface=en0 20 | 21 | # with listen-address, dnsmasq on OS X binds to UDP ports as follows: 22 | #listen-address={{ boot_server_ip }} 23 | listen-address=10.2.0.200 24 | # 67 (DHCP) on * 25 | # 69 (TFTP) on listen-address IP 26 | # 4011 (PXE proxyDHCP) on * 27 | 28 | # bind-interfaces disallows binding to 0.0.0.0, because dnsmasq doesn't support that on OS X 29 | # this avoids a warning in the log 30 | bind-interfaces 31 | 32 | # Make DHCP run in proxy mode, it does not supply IP addresses, only PXE responses 33 | #dhcp-range={{ boot_server_ip_base }},proxy,{{ boot_server_netmask }} 34 | dhcp-range=10.2.0.0,proxy,255.255.255.0 35 | 36 | # Relocate the dnsmasq.leases file which is created but not used 37 | #dhcp-leasefile={{ inventory_dir }}/.bootcfg/dnsmasq.data/dnsmasq.leases 38 | dhcp-leasefile=/Users/spencer/.bootcfg/assets/dnsmasq.data/dnsmasq.leases 39 | 40 | # Enable the TFTP server 41 | enable-tftp 42 | tftp-root=/Users/spencer/.bootcfg/assets/pxe 43 | 44 | # kill multicast for DHCP PXE 45 | dhcp-option=vendor:PXEClient,6,2b 46 | 47 | # set tag "ipxe" if request comes from iPXE ("iPXE" user class) 48 | dhcp-userclass=set:ipxe,iPXE 49 | 50 | # set tag "bzimage" if IPXE bzimage option supported 51 | dhcp-match=set:bzimage,175,24 52 | 53 | # if PXE request came from regular PXE firmware, serve iPXE firmware (via TFTP) 54 | dhcp-boot=tag:!ipxe,undionly.kpxe 55 | pxe-service=tag:!ipxe,x86PC,"PXE chainload to iPXE",undionly.kpxe 56 | 57 | # if PXE request came from PXE, or IPXE that does not support bzimage, serve iPXE firmware 58 | dhcp-boot=tag:!bzimage,undionly.kpxe 59 | pxe-service=tag:!bzimage,x86PC,"Load IPXE",undionly.kpxe 60 | 61 | # otherwise, grab an iPXE boot script from the bootcfg server 62 | #dhcp-boot=tag:ipxe,http://{{ boot_server_ip }}:8080/boot.ipxe 63 | dhcp-boot=tag:bzimage,http://10.2.0.200:8080/boot.ipxe 64 | #pxe-service=tag:ipxe,x86PC,"Run iPXE boot",http://{{ boot_server_ip }}:8080/boot.ipxe 65 | pxe-service=tag:bzimage,x86PC,"Run iPXE boot",http://10.2.0.200:8080/boot.ipxe -------------------------------------------------------------------------------- /docs/initial-setup.markdown: -------------------------------------------------------------------------------- 1 | # Initial setup 2 | 3 | ## Machines 4 | 5 | You will need: 6 | 7 | * an OS X or CoreOS machine to act as a PXE boot server. 8 | * machines that you want to PXE boot into CoreOS. We will call these the booted machines. 9 | * administrative access to the boot server. 10 | * physical access, most likely, to the booted servers. 11 | * All these machines should have Ethernet wired connections to the local network. 12 | 13 | ## Local network 14 | 15 | Your local network should have wired Ethernet connections available for your boot server and your booted machines. 16 | The network must have Internet access, typically via a NAT router. 17 | The network should have a DHCP service, and you need administrative access to the DHCP server. 18 | In many cases, this is simply your network's NAT router. 19 | 20 | You will need to assign static IP addresses to the boot server and the booted machines. So create an IP addressing plan for this. 21 | 22 | The best way to do this (and what is assumed here) is to have a DHCP server that supports DHCP reservations (aka static DHCP), which allows you to assign a fixed IP address to a machine's MAC address. 23 | When a machine boots, it will ask the DHCP server for an IP address, but will always get the reserved static IP address. 24 | 25 | Configure the DHCP server to assign a static IP to your boot server, then reboot the machine or refresh the DHCP lease to get the new IP address. 26 | 27 | ## Booted machines 28 | 29 | These machines should be Ethernet connected to your network, and be capable of PXE booting over the network. 30 | Most network adapters support PXE boot, and typically new machines come with PXE boot already set up as a default boot. 31 | You may need to boot a machine into its BIOS and reconfigure its boot settings for PXE booting. 32 | 33 | In some cases, the Ethernet port on the motherboard does not support PXE booting. 34 | The only recourse is to add a network adapter that does support PXE boot. 35 | 36 | ## Boot server setup 37 | 38 | The boot server must have the Go language installed and its environment set up. 39 | And also Python and Ansible. 40 | 41 | ### OS X 42 | 43 | Recommend using [Homebrew](http://brew.sh) 44 | 45 | `brew install go`. Then [follow instructions](https://golang.org/doc/install#testing) to set up your Go project directory and test your installation. 46 | 47 | `brew install python` and `pip install ansible`. 48 | 49 | `git clone https://github.com/SpencerBrown/coreos-pxe-install.git`. 50 | Or you may wish to fork the repo so you have a place to push your environments and configurations and share them with others. 51 | 52 | ### CoreOS setup 53 | 54 | You can install Go directly on CoreOS, no need to use containers. Logged in as user `core`: 55 | 56 | ```bash 57 | # Substitute a newer version of Go for "1.5.3" as appropriate 58 | wget https://storage.googleapis.com/golang/go1.5.3.linux-amd64.tar.gz 59 | tar -xzf go1.5.3.linux-amd64.tar.gz 60 | cp .bashrc bashrc 61 | echo 'export GOROOT=~/go' >> bashrc 62 | echo 'export PATH=$PATH$GOROOT/bin' >> bashrc 63 | echo 'export GOPATH=$HOME/work' >> bashrc 64 | rm .bashrc 65 | mv bashrc .bashrc 66 | source .bashrc 67 | ``` 68 | 69 | TODO: install PyPY and Ansible directly on CoreOS. 70 | 71 | ### Download and build projects 72 | 73 | ```bash 74 | go get -d github.com/coreos/coreos-baremetal/cmd/bootcfg 75 | cd $GOPATH/src/github.com/coreos/coreos-baremetal 76 | ./build 77 | ``` 78 | 79 | The binary `bootcfg` is built for your OS and saved in `$GOPATH/src/github.com/coreos/coreos-baremetal/bin`. 80 | 81 | OS X: `brew install dnsmasq` 82 | 83 | CoreOS: TBD -------------------------------------------------------------------------------- /roles/cbm/meta/main.yml: -------------------------------------------------------------------------------- 1 | galaxy_info: 2 | author: your name 3 | description: 4 | company: your company (optional) 5 | 6 | # If the issue tracker for your role is not on github, uncomment the 7 | # next line and provide a value 8 | # issue_tracker_url: http://example.com/issue/tracker 9 | 10 | # Some suggested licenses: 11 | # - BSD (default) 12 | # - MIT 13 | # - GPLv2 14 | # - GPLv3 15 | # - Apache 16 | # - CC-BY 17 | license: license (GPLv2, CC-BY, etc) 18 | 19 | min_ansible_version: 1.2 20 | 21 | # Optionally specify the branch Galaxy will use when accessing the GitHub 22 | # repo for this role. During role install, if no tags are available, 23 | # Galaxy will use this branch. During import Galaxy will access files on 24 | # this branch. If travis integration is cofigured, only notification for this 25 | # branch will be accepted. Otherwise, in all cases, the repo's default branch 26 | # (usually master) will be used. 27 | #github_branch: 28 | 29 | # 30 | # Below are all platforms currently available. Just uncomment 31 | # the ones that apply to your role. If you don't see your 32 | # platform on this list, let us know and we'll get it added! 33 | # 34 | #platforms: 35 | #- name: EL 36 | # versions: 37 | # - all 38 | # - 5 39 | # - 6 40 | # - 7 41 | #- name: GenericUNIX 42 | # versions: 43 | # - all 44 | # - any 45 | #- name: Solaris 46 | # versions: 47 | # - all 48 | # - 10 49 | # - 11.0 50 | # - 11.1 51 | # - 11.2 52 | # - 11.3 53 | #- name: Fedora 54 | # versions: 55 | # - all 56 | # - 16 57 | # - 17 58 | # - 18 59 | # - 19 60 | # - 20 61 | # - 21 62 | # - 22 63 | #- name: Windows 64 | # versions: 65 | # - all 66 | # - 2012R2 67 | #- name: SmartOS 68 | # versions: 69 | # - all 70 | # - any 71 | #- name: opensuse 72 | # versions: 73 | # - all 74 | # - 12.1 75 | # - 12.2 76 | # - 12.3 77 | # - 13.1 78 | # - 13.2 79 | #- name: Amazon 80 | # versions: 81 | # - all 82 | # - 2013.03 83 | # - 2013.09 84 | #- name: GenericBSD 85 | # versions: 86 | # - all 87 | # - any 88 | #- name: FreeBSD 89 | # versions: 90 | # - all 91 | # - 8.0 92 | # - 8.1 93 | # - 8.2 94 | # - 8.3 95 | # - 8.4 96 | # - 9.0 97 | # - 9.1 98 | # - 9.1 99 | # - 9.2 100 | #- name: Ubuntu 101 | # versions: 102 | # - all 103 | # - lucid 104 | # - maverick 105 | # - natty 106 | # - oneiric 107 | # - precise 108 | # - quantal 109 | # - raring 110 | # - saucy 111 | # - trusty 112 | # - utopic 113 | # - vivid 114 | #- name: SLES 115 | # versions: 116 | # - all 117 | # - 10SP3 118 | # - 10SP4 119 | # - 11 120 | # - 11SP1 121 | # - 11SP2 122 | # - 11SP3 123 | #- name: GenericLinux 124 | # versions: 125 | # - all 126 | # - any 127 | #- name: Debian 128 | # versions: 129 | # - all 130 | # - etch 131 | # - jessie 132 | # - lenny 133 | # - squeeze 134 | # - wheezy 135 | 136 | galaxy_tags: [] 137 | # List tags for your role here, one per line. A tag is 138 | # a keyword that describes and categorizes the role. 139 | # Users find roles by searching for tags. Be sure to 140 | # remove the '[]' above if you add tags to this list. 141 | # 142 | # NOTE: A tag is limited to a single word comprised of 143 | # alphanumeric characters. Maximum 20 tags per role. 144 | 145 | dependencies: [] 146 | # List your role dependencies here, one per line. 147 | # Be sure to remove the '[]' above if you add dependencies 148 | # to this list. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Status: DO NOT USE. In transition. 2 | 3 | Work items: 4 | 5 | 1. Stop using Ansible and start using Go language with templating instead. 6 | 2. Update due to lots of changes in the coreos-baremetal project. 7 | 3. Update because the iPXE driver for VirtualBox is now too large (see repo virtualbox-ipxe) 8 | 9 | 10 | # Network booting CoreOS to bare metal machines 11 | 12 | This project uses two excellent projects: [dnsmasq](http://www.thekelleys.org.uk/dnsmasq/doc.html) and [coreos-baremetal](https://github.com/coreos/coreos-baremetal). Both of these work on either Linux or OS X. 13 | 14 | This guide details how to set up an "environment", which is a cluster of CoreOS bare metal servers. 15 | Ansible playbooks are provided to automate the setup and operation. 16 | 17 | Testing have been done using OS X as the boot server, and VirtualBox virtual machines and real machines as the booted machines. 18 | 19 | # Initial setup 20 | 21 | See [Initial Setup document](docs/initial-setup.markdown). 22 | 23 | # Create an environment 24 | 25 | To create an environment, pick a name that's unique in your repository. For this example let's call it `vbox`. 26 | The environment represents a CoreOS cluster of bare metal machines, which share an SSH key and a configuration. 27 | 28 | `ansible-playbook -e env=vbox -i hosts make-environment.yaml` will create your environment for you, 29 | including a new SSH keypair `vbox-key` and `vbox-key.pub`. 30 | This keypair is saved in your `~/.ssh` directory, and also copied to your environment's `secret-files` and `public-files` directories. 31 | 32 | Once your environment is set up, change to its directory, for example, `cd vbox`. 33 | 34 | ## Sharing an environment 35 | 36 | If you wish to share your environment, configurations, and secrets with others, create a fork of the repo. 37 | Do your setup, then encrypt your secrets and push the results to your fork on GitHub. 38 | Another team member can clone your repo and add the environment's password, then decrypt the secrets. 39 | See [Sharing Secrets](docs/sharing-secrets.markdown) for more information. 40 | 41 | # Configure your environment 42 | 43 | Change to your environment directory, and edit the file `group_vars/local/coreos.yaml`. 44 | Set the variables according to your local configuration as follows: 45 | 46 | Variable | Value | Default 47 | ---------|-------|-------- 48 | boot_server_ip | IP address of boot server | 10.2.0.200 49 | boot_server_ip_base | Base IP address of local network | 10.2.0.0 | 50 | boot_server_ip_netmask | Netmask for local network | 255.255.255.0 51 | coreos_channel | CoreOS release channel (alpha, beta, stable) | alpha 52 | coreos_release | CoreOS specific release e.g. 935.0.0 | current 53 | 54 | Setting coreos_release to "current" will fetch the latest release for that release channel. 55 | 56 | Then, run `ansible_playbook ../make_cbm.yaml` to configure your environment and download the necessary binaries. 57 | If anything changes, rerun the playbook to reconfigure the environment. 58 | 59 | # Start the boot services 60 | 61 | Set the environment variable CBM to the path for the `cbm` directory for your environment. For example: 62 | 63 | `export CBM=~/src/coreos-pxe-install/vbox/cbm` 64 | 65 | ## OS X 66 | 67 | Start the coreos-baremetal service: 68 | 69 | ```bash 70 | cd $CBM 71 | ./bootcfg --address=0.0.0.0:8080 72 | ``` 73 | 74 | In another terminal window, start the dnsmasq service: 75 | 76 | ```bash 77 | cd $CBM 78 | sudo ./dnsmasq -C dnsmasq.data/dnsmasq.conf -k --log-facility=- --log-dhcp 79 | ``` 80 | 81 | dnsmasq must run as root because it opens privileged ports to listen for DHCP and TFTP requests from the PXE booted servers. 82 | You must enter your OS X password. 83 | 84 | # Boot your booted machine for the first time 85 | 86 | Power on the booted machine. For first time use, you may want to attach a keyboard and display for debugging purposes. 87 | You should see CoreOS boot up with the hostname set to `default`. 88 | If you have a keyboard/display attached, CoreOS will automatically login the core user. 89 | 90 | SSH into the machine. Commands to use: 91 | * `ip addr` - discover IP addresses and MAC addresses. 92 | * `lsblk` - discover disks. 93 | 94 | ## Troubleshooting 95 | 96 | The most common cause of network boot failure is incorrect BIOS configuration on the booted server. 97 | 98 | ## Note the server MAC address 99 | 100 | The window running `dnsmasq` will have a message similar to: 101 | 102 | `dnsmasq-dhcp[8655]: PXE(en0) 90:2b:34:14:f6:6a proxy` 103 | 104 | Make a note of the MAC address of your booted machine, in this example it is `90:2b:34:14:f6:6a` 105 | 106 | 107 | -------------------------------------------------------------------------------- /docs/manual-setup.markdown: -------------------------------------------------------------------------------- 1 | # How to network boot CoreOS machines from an OS X or CoreOS server 2 | 3 | NOTE: this guide is for manual setup. The current project has automation of the setup using Ansible. See `README.md`. 4 | 5 | This guide uses two excellent projects: [dnsmasq](http://www.thekelleys.org.uk/dnsmasq/doc.html) and [coreos-baremetal](https://github.com/coreos/coreos-baremetal). Both of these work on either Linux or OS X. 6 | This guide details how to install and use them to set up a fast, flexible network boot environment on OS X. 7 | 8 | It is possible to use older projects like [dnsmasq](http://www.thekelleys.org.uk/dnsmasq/doc.html) to accomplish this goal, but setup is much more complex and less flexible. 9 | 10 | ## Create your PXE boot environment 11 | 12 | ### Create the config directory 13 | 14 | ```bash 15 | cd ~ 16 | mkdir cbm 17 | export CBM=~/cbm 18 | cd $CBM 19 | mkdir -p {data/ignition,data/cloud,data/specs/first-time} 20 | mkdir -p dnsmasq.data/tftp 21 | ln -s $GOPATH/src/github.com/coreos/coreos-baremetal/bin/bootcfg bootcfg 22 | ln -s $GOPATH/src/github.com/coreos/coreos-baremetal/scripts/get-coreos get-coreos 23 | ln -s /usr/local/sbin/dnsmasq dnsmasq 24 | ``` 25 | 26 | ### Download the CoreOS PXE binaries 27 | The following downloads a version of CoreOS for PXE booting. 28 | Modify "alpha" and "current" to the channel and release that you wish to use. 29 | 30 | ```bash 31 | ./get-coreos alpha current 32 | ``` 33 | 34 | ### Create the SSH keypair 35 | 36 | Create an SSH keypair for your PXE boot servers using `ssh-keygen`. In this example we'll call it `coreos-pxe`. 37 | 38 | ```bash 39 | cd ~/.ssh 40 | ssh-keygen -f coreos-pxe 41 | # hit enter to take all defaults 42 | ``` 43 | 44 | ### Create the first-time ignition config file 45 | 46 | Create `$CBM/data/ignition/first-time.json` adding your SSH public key file contents: 47 | 48 | ```json 49 | { 50 | "ignitionVersion": 1, 51 | "passwd": { 52 | "users": [ 53 | { 54 | "name": "core", 55 | "sshAuthorizedKeys": [ 56 | "PASTE YOUR SSH PUBLIC KEY HERE: CONTENTS OF FILE coreos-pxe.pub" 57 | ] 58 | } 59 | ] 60 | }, 61 | "systemd": { 62 | "units": [ 63 | { 64 | "name": "set-hostname.service", 65 | "enable": true, 66 | "contents": "[Unit]\n[Service]\nType=oneshot\nExecStart=/usr/bin/hostnamectl set-hostname first-time\n[Install]\nWantedBy=multi-user.target" 67 | } 68 | ] 69 | } 70 | } 71 | ``` 72 | ### Create the first-time spec 73 | 74 | Create `$CBM/data/spec/first-time/spec.json` as follows: 75 | 76 | * substitute the boot server's IP address for `10.2.0.200` 77 | * if you downloaded a specific CoreOS version, substitute that version number for `current` 78 | 79 | ```json 80 | { 81 | "id": "first-time", 82 | "boot": { 83 | "kernel": "/images/coreos/current/coreos_production_pxe.vmlinuz", 84 | "initrd": [ 85 | "/images/coreos/current/coreos_production_pxe_image.cpio.gz" 86 | ], 87 | "cmdline": { 88 | "coreos.config.url": "http://10.2.0.200:8080/ignition?uuid=default", 89 | "coreos.autologin": "", 90 | "coreos.first_boot": "1" 91 | } 92 | }, 93 | "ignition_id": "pxe-ignition-default.json" 94 | } 95 | ``` 96 | 97 | ### Create the dnsmasq config files 98 | 99 | FOr OS X, create `$CBM/dnsmasq.data/dnsmasq.conf` as follows: (substitute your OS X username for `spencer` 100 | 101 | ``` 102 | # Disable DNS service 103 | port=0 104 | 105 | # Set the username that dnsmasq will switch to after startup 106 | # note that dnsmasq must be started as root 107 | user=spencer 108 | 109 | # with no interface=, dnsmasq on OS X binds to UDP ports as follows: 110 | # 67 (DHCP) on * 111 | # 69 (TFTP) on adapter's IP e.g. 10.2.0.200 and also 127.0.0.1 112 | # 4011 (PXE proxyDHCP) on * 113 | # also a bunch of IPv6 listent 114 | #interface=en0 115 | 116 | # with listen-address, dnsmasq on OS X binds to UDP ports as follows: 117 | listen-address=10.2.0.200 118 | # 67 (DHCP) on * 119 | # 69 (TFTP) on listen-address IP 120 | # 4011 (PXE proxyDHCP) on * 121 | 122 | # bind-interfaces disallows binding to 0.0.0.0, because dnsmasq doesn't support that on OS X 123 | # this avoids a warning in the log 124 | bind-interfaces 125 | 126 | # Make DHCP run in proxy mode, it does not supply IP addresses, only PXE responses 127 | dhcp-range=10.2.0.0,proxy 128 | 129 | # Relocate the dnsmasq.leases file which is created but not used 130 | dhcp-leasefile=/Users/spencer/pixie/dnsmasq.data/dnsmasq.leases 131 | 132 | # Enable the TFTP server 133 | enable-tftp 134 | tftp-root=/Users/spencer/pixie/dnsmasq.data/tftp 135 | 136 | # kill multicast for DHCP PXE 137 | dhcp-option=vendor:PXEClient,6,2b 138 | 139 | # set tag "ipxe" if request comes from iPXE ("iPXE" user class) 140 | dhcp-userclass=set:ipxe,iPXE 141 | 142 | # if PXE request came from regular PXE firmware, serve iPXE firmware (via TFTP) 143 | dhcp-boot=tag:!ipxe,undionly.kpxe 144 | pxe-service=tag:!ipxe,x86PC,"PXE chainload to iPXE",undionly.kpxe 145 | 146 | # if PXE request came from iPXE, grab an iPXE boot script from the bootcfg server 147 | dhcp-boot=tag:ipxe,http://10.2.0.200:8080/boot.ipxe 148 | pxe-service=tag:ipxe,x86PC,"Run iPXE boot",http://10.2.0.200:8080/boot.ipxe 149 | ``` 150 | 151 | ## Set up for subsequent boots of booted machine 152 | 153 | ### Set DHCP reserved IP address for machine 154 | 155 | On your DHCP service, use the MAC address noted above to set up a static reserved IP for the machine. 156 | 157 | ### Set up configuration for the machine 158 | 159 | Create a directory for this machine, named with the MAC address: 160 | 161 | ```bash 162 | # Obviously, substitute the correct MAC address # 163 | mkdir $CBM/data/90:2b:34:14:f6:6a 164 | ``` 165 | 166 | Inside this directory, create a `machine.json` file for that machine: 167 | 168 | ### Reboot the machine 169 | 170 | It should boot up with the reserved IP address, and the new PXE boot configuration. 171 | 172 | ### SSH into server 173 | 174 | Use your private SSH key to login remotely to the server, using the static IP address you assigned. 175 | The username is `core`. 176 | 177 | `ssh -i ~/.ssh/coreos-pxe core@10.2.0.197` 178 | 179 | ### Clean out disks 180 | 181 | ```bash 182 | sudo gdisk /dev/sda 183 | # use "o" to completely wipe the disk 184 | # use "n" to create a new partition 185 | ``` 186 | 187 | # Notes 188 | 189 | CoreOS PXE boot with kernel parameter `root=/dev/sda1` to an empty disk sets it up as follows: 190 | 191 | Populated directories: 192 | 193 | ``` 194 | /etc 195 | /var 196 | ``` 197 | 198 | Directories that exist, but are mounted with `tmpfs` file systems: 199 | 200 | ``` 201 | /run 202 | /media 203 | /tmp 204 | ``` 205 | 206 | `/usr` is a special case. It is mounted as a squashfs file system pointing to `usr.squashfs`. So it runs out of memory, I guess. 207 | `/dev/loop0` is mounted on `/usr` by the `usr.mount` service. -------------------------------------------------------------------------------- /docs/utility-machine.markdown: -------------------------------------------------------------------------------- 1 | # Create an Arch Linux utility machine 2 | 3 | NOTE: this document is not applicable to the current setup. We are not using Arch Linux. 4 | We are keeping this document here because it's a nice guide to setting up Arch Linux in general. 5 | 6 | ## Acquire a machine 7 | 8 | It should: 9 | 10 | * be capable of running on its own without a keyboard/mouse 11 | * have two network adapters, one set up for DHCP-fueled access to the Internet, one facing inward to the cluster it's managing 12 | * have a reasonably new Intel 64-bit multicore CPU 13 | * have a reasonably modern motherboard that supports UEFI boot 14 | * have a reasonable size/number of disk drives. SSDs are very nice. 15 | * have at least 4GB memory 16 | 17 | ## Installing Arch Linux 18 | 19 | Generally, follow the directions in the [Beginners' Guide](https://wiki.archlinux.org/index.php/Beginners'_guide) or the [Installation Guide](https://wiki.archlinux.org/index.php/Installation_guide). 20 | 21 | ### Create the install media 22 | 23 | 1. Download the Arch Linux ISO from [Rackspace](http://mirror.rackspace.com/archlinux/iso) or another [mirror site](https://wiki.archlinux.org/index.php/Mirrors). 24 | 2. If installing on a real machine, burn the ISO to a USB flash drive or CD-R disk. 25 | 26 | #### How to burn the .iso file to a USB flash drive using OS X 27 | 28 | 1. Run `diskutil list`, then insert the flast drive and run `diskutil list` again. 29 | 2. Figure out its device name by noting the differences in the lists. 30 | 3. Let's say its device name is `/dev/disk2`, adjust the following as needed. 31 | 4. `diskutil unmountDisk /dev/disk2` to unmount the mounted partitions. 32 | 5. NOTE THE USE OF `/dev/rdisk2` instead of `/dev/disk2` in the next step. 33 | 6. VERY CAREFULLY: MAKE SURE YOUR DEVICE NAME FOR `of=` IS CORRECT OR YOU MAY DESTROY YOUR SYSTEM: 34 | 7. `sudo dd if=~/Downloads/archlinux-2015.12.01-dual.so of=/dev/rdisk2 bs=1m` 35 | 8. `diskutil eject /dev/disk2` 36 | 9. Remove the drive. 37 | 38 | ### Install basic Arch Linux 39 | 40 | Connect (temporarily) a keyboard and display to your machine. Boot the machine into its BIOS settings. (Typically you press the Delete or F2 key several times after booting.) 41 | 42 | Or, many machines now support a special "boot override" mode which does a one-time boot from a specified device, without changing the BIOS defaults. 43 | Try F11 for this, or see the display on your screen from the BIOS for a hint. 44 | 45 | Adjust the machine's BIOS settings to boot from your media, selecting the UEFI boot if available. 46 | 47 | Select the 64-bit version to run, if you are given a choice. 48 | 49 | #### Initial setup 50 | 51 | `ping google.com` to ensure you have Internet connectivity. 52 | 53 | `timedatectl set-ntp true` to sync your clock. 54 | 55 | Run `lsblk` and decide which disk is going to be your boot drive. It will be completely overwritten. 56 | 57 | #### Partition and format the boot drive 58 | 59 | In this example the boot drive is `/dev/sda`, adjust accordingly. It will be completely overwritten, all existing data will be lost. 60 | 61 | We will create two partitions. Partition 1 is the EFI system partition for booting Arch Linux via UEFI boot. 62 | Partition 2 is a Linux partition where the system resides. 63 | 64 | to reset the disk and create the EFI system partition: 65 | ``` 66 | sgdisk --zap-disk /dev/sda 67 | gdisk /dev/sda 68 | o (overwrite partition table) 69 | n (create new partition) 70 | (enter) (accept default partiton 1) 71 | (accept default start) 72 | 512M (partition size 512 megabytes) 73 | ef00 (partition type EFI System) 74 | w (write to disk) 75 | y (proceed) 76 | mkfs.fat -F32 /dev/sda1 77 | ``` 78 | 79 | to create the Linux partition: 80 | 81 | ``` 82 | gdisk /dev/sda 83 | n (create new partition) 84 | 4 times (accept all the defaults) 85 | w (write to disk) 86 | y (proceed) 87 | mkfs.ext4 /dev/sda2 88 | ``` 89 | 90 | #### Set up mirror list 91 | 92 | The mirror list determines where Arch Linux installation will go to download packages. 93 | You can use the mirror list as is but it will probably use a very slow website. 94 | 95 | ``` 96 | cp /etc/pacman.d/mirrorlist /etc/pacman.d/mirrorlist.original (save original mirror list) 97 | vim /etc/pacman.d/mirrorlist 98 | %s:/Server =/#Server =/ (comment out all the servers) 99 | (now locate servers you like and uncomment those lines... rackspace is usually a good choice) 100 | :wq 101 | ``` 102 | 103 | #### Download packages 104 | 105 | ``` 106 | mount /dev/sda2 /mnt (adjust partition name as needed) 107 | mkdir /mnt/boot 108 | mount /dev/sda1 /mnt/boot 109 | pacstrap /mnt base base-devel (this will download and install a bunch of packages) 110 | ``` 111 | 112 | #### Basic configuration of the new system 113 | 114 | This sets up the disk mount table, clock, timezone, and locale. 115 | Then we set up the network adapter, and basic configuration of the UEFI boot partition. 116 | 117 | ``` 118 | genfstab -p /mnt >> /mnt/etc/fstab (generate initial mount table) 119 | arch-chroot /mnt (enter the new system's directory tree) 120 | echo utility > /etc/hostname (sub your desired hostname for "utility") 121 | ln -s /usr/share/zoneinfo/US/Central /etc/localtime (set local time zone, modify "Central" as appropriate) 122 | hwclock --systohc --utc (set hardware clock to system time in UTC) 123 | vi /etc/locale.gen (set default locale) 124 | (Uncomment line "# en_US.UTF-8 UTF-8", and save) 125 | locale-gen (generate the new locale information) 126 | echo LANG=en_US.UTF-8 > /etc/locale.conf (set the default locale) 127 | ip addr (list installed network adapters) 128 | (pick the adapter that's connected to the Internet, we use "enp4s0" in this example) 129 | systemctl enable dhcpcd@enp4s0.service (enable DHCP on adapter at boot) 130 | mkinitcpio -p linux (create Linux initial boot binary) 131 | passwd (set a root password) 132 | (enter a root password twice) 133 | bootctl install (install UEFI boot structure to /boot) 134 | exit (return to the base system) 135 | ``` 136 | 137 | #### Create boot configuration 138 | 139 | `vim /mnt/boot/loader/entries/arch.conf` to create a new boot configuration file with the following contents. 140 | Of course, adjust `/dev/sda2` to reflect your boot partition. 141 | 142 | ``` 143 | title Arch Linux 144 | linux /vmlinuz-linux 145 | initrd /initramfs-linux.img 146 | options root=/dev/sda2 rw 147 | ``` 148 | 149 | `vim /mnt/boot/loader/loader.conf` to point the EFO bootloader to the new `arch.conf`. 150 | 151 | Replace the line starting with `default` with: 152 | 153 | `default arch` 154 | 155 | #### Boot the new system 156 | 157 | Remove the USB drive and `systemctl reboot`. You may need to enter BIOS and adjust the boot settings to boot to the Linux bootloader. 158 | 159 | #### Alternative using old style disk partitioning 160 | 161 | Instead of the "Partition and format the boot drive" above, do this: 162 | 163 | ``` 164 | # partition and format the boot disk in old school fashion, use the appropriate device name throughout 165 | fdisk /dev/sda 166 | # n 167 | # (press enter several times) 168 | # w 169 | mkfs.ext4 /dev/sda1 170 | mount /dev/sda1 /mnt 171 | ``` 172 | 173 | Instead of "Create book configuration" above, do this: 174 | 175 | ``` 176 | pacman -S grub os-prober 177 | grub-install --recheck /dev/sda 178 | grub-mkconfig -o /boot/grub/grub.cfg 179 | ``` 180 | 181 | ## Configuring Arch Linux 182 | 183 | 1. Login as `root`. 184 | 185 | ``` 186 | pacman -S pkgfile 187 | pkgfile --update (pkgfile is useful to find out what packages contains a command file) 188 | pacman -S vim (installs vim, also ruby, python2, and lua) 189 | ln -s /usr/bin/python2 /usr/bin/python (makes the "python" command use Python 2. We don't need no stinking Python 3.) 190 | useradd -m -G wheel -s /bin/bash admin (adds user admin, pick your own username if you want) 191 | passwd admin 192 | (enter password for admin user) 193 | EDITOR=vim visudo 194 | (uncomment the line "# %wheel ALL=(ALL) NOPASSWD: ALL") 195 | ``` 196 | 197 | ## Configuring the network 198 | 199 | As root: 200 | 201 | ``` 202 | systemctl enable systemd-networkd 203 | systemctl start systemd-networkd 204 | systemctl enable systemd-resolved 205 | systemctl start systemd-resolved 206 | ln -sf /run/systemd/resolve/resolv.conf /etc/resolv.conf 207 | ``` 208 | 209 | ### Recycling DHCP lease after setting fixed IP address on DHCP server 210 | 211 | As root: 212 | 213 | ``` 214 | rm /var/lib/dhcpcd/dhcpcd-.lease 215 | systemctl reboot 216 | ``` 217 | 218 | ## Setting up SSH access 219 | 220 | As root: 221 | 222 | ``` 223 | pacman -S openssh 224 | systemctl enable sshd 225 | systemctl start sshd 226 | ``` 227 | 228 | As user `admin`: `mkdir ~/.ssh` 229 | 230 | Now go to another machine: (can be OS X or Linux) 231 | 232 | ``` 233 | ssh-keygen -f ~/.ssh/utility -N "" (or whatever finename you wish besides "utility") 234 | scp ~/.ssh/utility.pub admin@:.ssh/authorized_keys 235 | ``` 236 | 237 | You can now login to the utility machine by running `ssh -i ~/.ssh/utility admin@`. 238 | 239 | Now lock down SSH on the machine by disabling root access, and password login: 240 | 241 | ``` 242 | sudo vim /etc/ssh/sshd_config 243 | # uncomment line #PasswordAuthentication and change "yes" to "no" 244 | # uncomment line #AllowAgentForwarding yes 245 | # uncomment line #PermitRootLogin and change parameter to "no" 246 | sudo systemctl restart sshd 247 | ``` 248 | 249 | ### Setting up a second adapter with a static IP 250 | 251 | As root, create a network unit called `internal.network` like this in `/etc/systemd/network`: 252 | (use the correct device name) 253 | 254 | ``` 255 | [Match] 256 | Name=enp2s0 257 | [Network] 258 | Address=10.0.0.1/16 259 | ``` 260 | 261 | Then, `systemctl restart systemd-networkd` to enable it. 262 | 263 | ## Mounting additional disks at boot 264 | 265 | As root, put a mount unit like this into `/etc/systemd/system`, and call it `disk1.mount`. 266 | Then `systemctl enable disk1.mount` and `systemctl start disk1.mount`. 267 | 268 | ``` 269 | [Unit] 270 | Description=Mount Disk 1 271 | Before=local-fs.target umount.target 272 | Conflicts=umount.target 273 | DefaultDependencies=no 274 | [Mount] 275 | What=/dev/sdb1 276 | Where=/disk1 277 | [Install] 278 | WantedBy=local-fs.target 279 | ``` 280 | 281 | ## Setting up Wake On LAN for your utility machine 282 | 283 | [See here for advice]() on enabling Wake On Lan on the utility machine. I had to enable "PCIE Devices Power On" in one of the BIOS menus. 284 | 285 | Then, `brew install wakeonlan` on your Mac. Find the adapter MAC address from `ip addr`, and save it on your Mac. 286 | 287 | `wakeonlan ` will then power up your machine from anywhere on the LAN. 288 | 289 | ## Updating your utility machine 290 | 291 | `sudo pacman -Syu` will update all packages on your machine. You will need to reboot if the Linux kernel is updated. -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | 203 | -------------------------------------------------------------------------------- /vbox/group_vars/local/vault.yaml: -------------------------------------------------------------------------------- 1 | $ANSIBLE_VAULT;1.1;AES256 2 | 37303134316266303266353962356535343963346638643363656338393466613439346130393235 3 | 3666336135323231306130633138323162346163303732310a626236336130646563313863323139 4 | 61666462643836653339656634653136626636623139386137353066313166643935663135653461 5 | 3132623235336139620a376438366334363633306330643961613336373738613439616335336262 6 | 31363532346637336666336331653533656534393862393031666239633939616234393431656131 7 | 36366138613561326134613735646139386139363066303762623136343265663830373966363731 8 | 33363639333134313661363134373636313363393432633231666162663765353137643439316263 9 | 66323232323665336235643639383332316363353330613734643930316664666135363832636261 10 | 31316662303163343262626539666639643737316231396335303431323965336334376562333564 11 | 66353339373963613634623264373063323830356563656262316639656166633566333733333533 12 | 64636134303963636338376661376262313336613366386434353562653936373564376431656334 13 | 61646336396433303131393131613238343863636437643639373536353166663533633536303435 14 | 62323933626235626564613433616366663863303366653962346136323762616337333564613662 15 | 38383966663031313965353863616161613561626138313036316334323465653135366132353562 16 | 62616636626131656162326234343838366565626266343566623531333331353937663065386266 17 | 33356435363463336135366234303930333366333234393064613664363334383934306339633866 18 | 61633834356365633839346631666636636137323135373863393262323032333132636662666331 19 | 31313965656463386361396262633562323562633532636336353531653861623233373462383630 20 | 64343165643132323737376439633834666533313961373338663665313035333166393764303564 21 | 33646631363161666661613031613636663430643134326339326131326238626631633630366439 22 | 33663764646336376362646161343437343530626162346435323066383334666462363332656331 23 | 35333936383961663934316363386163326532663337663036633765623563353437396666343965 24 | 65636565313837316534636235623738656461646139613835346338373633353065643432616432 25 | 63653035613939616432353737316661613164383963653763616637326339333138303932346539 26 | 30613462656265633433666262323132323537326363316663316435653033323037666233383138 27 | 65623462323339323165393966663362643934666339366365623539613333656333353331656438 28 | 34373066343161666662383533346430383230653330663132663963613961383736613163353732 29 | 62333562333361306261643135333034376364323133346364393266376266326361343337306434 30 | 64356666386237663063616339383835393336316261306233353964626634373230393961373464 31 | 34633837376164333264393233383838373733626331393138653638353465313364663037356664 32 | 33663833646633393134326534383435383133376134613931336437353737353736616233356463 33 | 64626162326666346635356564363830306661646464393661343931383638336530333461366435 34 | 35376537653937306366346232656234326130636164333939346334356462323463363562346230 35 | 64373736616531363761306434356334343963383730656439643363356335333661366361643165 36 | 37623933313862633238376230616465336561346165646266376364653731646239316131663464 37 | 61363962646534363634666437343962646464383935633130383835313133393832353337386432 38 | 38623831343838363263386336313765313331336334383736346433613736373635376630333532 39 | 34633435656164393833333436393533376137643165323436616661626232343265393661656438 40 | 65636133396237363637346664623966343936353830383734656563646165653037343635646231 41 | 65656531306630623964383266356365643837643938313734633436643661336565323961643432 42 | 35626364646366326662396561393363373533656264326331366564613831613932386233353363 43 | 66653439653930336134626531346339363862323065383736373165643530303937633163623139 44 | 63643237353731363033323065313236636662633636323635393939393533663462363435626262 45 | 62383537616336613737316533626263353638353838343130353032666432323330616662386338 46 | 36633230376436393438376232613939623965323033306635653531306564336430666362313434 47 | 64663463376363393434626463393064323265613235663631643432326135663437366430303932 48 | 66363933393734623230333939616663616434623835393232656539313933666131643163396337 49 | 31366339636239386365616332363166363666383765626233643137376463663936613937633638 50 | 37643161343930316562623365633130363562326361353966613337333864663035663065303236 51 | 31363539613238613737353863366137636135323930643464656461326630653963333130613233 52 | 65353035393962393030363137356462656465376261343464643235306639343431613062636264 53 | 61666635393766643563643639333161643063383430396161656161666536366361636336323839 54 | 66626535383332326238393837326431313664306264313537313663646437363063633538383366 55 | 39363061383135383834613861626630636330353362346636396632653665656431303030646634 56 | 61363564663639623537386131633534623538666134656365633237366630316430646663623237 57 | 61656130643365313631343665666165306636366333323663653136373930666332356563373366 58 | 32616463656139316231633263376630303035333661636163313762326466346538333439383561 59 | 62663831666665343965346666633065363735336163643732393066353365343062333934323233 60 | 65353034313734653231613431333836353862373531653937333639373363316631363735626661 61 | 37653564356636613662386364306666643139633538616438316632343834343266306230666638 62 | 30343331613864303034313866626466343033303232343738633132633832346434333935323463 63 | 32303133393436383165313736343737353038346131356461333931376137383939663161613736 64 | 31333337376562653034616130623035633137363961646536613463353130623336303865383334 65 | 30303266323863343965626435343865616661346639306630383735356264366430386361623832 66 | 34383338313032633166656639646562666438623139336262356234643137326533656336373835 67 | 66383764373663356335303962373063613937373766336330396563646664356331666532646633 68 | 64366361343833643037383463626133353164313733366463613162346233616362316631663230 69 | 66616262626666373563386634373066346535616565666530386430306339653636326132323635 70 | 36336133356133623863626333356536353034643566653832333139666534383239663439633562 71 | 39393466393166633232666530373936393636643230303038646538356161613936663865393764 72 | 39363238303066623266376539663537666139666534386638326139643666376132356432323262 73 | 32393733663063343036613637396238316135326464313339383039376330373438636431373737 74 | 36663061643030613139343965633138303138633835373834383636636332393361633665646361 75 | 30613834616434343639363166633030653430643261633434353732396135323361346661316231 76 | 37653830646334333630623433613937666366656239343137616263346639646638346233386161 77 | 36633565613262346230373964363435333730643735313637663765396436376238313930383436 78 | 61653238346666373663343661626363623864653764653334383565363165616164663762636561 79 | 63383230373665366430343462326232303064303663666264636538623139646532386365633438 80 | 37326138363962363833303938623264653765323035383932663933646535613032646264663962 81 | 39626437326264363763633533363131613737616261613836316430326638613933643534363633 82 | 35376631373238366437353265396534663434646530663663333434396365306462346430626363 83 | 30643863356464393466616662396437366633386264656532353833663130343264663836306463 84 | 32393931623062303130326537636562663839653237646466623332343635376134666130346330 85 | 39626334666664303663636137663666376537666430333265333462633234626466636231653038 86 | 31613162636430306131336234613231316432303264636438616666663133323366363762613065 87 | 39663034373532666134633036663438373536353961373862353965363339373736303838643061 88 | 35393862366164663162366262383030666134386430393836353532363432376334313438643338 89 | 38633265333833653431626366336539663665646634353539643261386563633838633432333966 90 | 65646436373635613461363765313863343335313663336539623661323166396431353662336563 91 | 31356634623630363439356237663731393633656230626361623332393239623061626631643731 92 | 32393938373863373032326162353732306333386235356636633962636534303339656464646435 93 | 39373632363765363435383661363837653366613132643331303838316131343566343537646361 94 | 66393738386539373064633861353831323338393833653634396132643863383636313237653030 95 | 30626562373838346331636564353638643062623061313039336530633733326166356262343537 96 | 33656534366135623236313330313132633231313962333337633465353663666535646438353661 97 | 62393661323162396465663335326365636237623866306262323733313234396536623562626634 98 | 63653762613833616462313736343238356165346261393633643861626139376261346261333134 99 | 61356463343465623037356138366432643761313036316461323839666137396431626637633837 100 | 61353530373039326565613235303538333431326465653637316562376565643665306338613361 101 | 64656336393163666361353132313534613034653137363765376161343965393261363530393061 102 | 30656637626463616436316464323962383034633833373037343832653935623363383663303330 103 | 36386130363337366162663332356239343534353633313538346237313930663638336335653561 104 | 32663634356331383263376664396437346431353564343634356237393437336136353438316131 105 | 34313335396338663338643030356565353963663135386133333030613564303865303262346130 106 | 66383537363864643236393064363431316230306230363664313834313335313031336534323934 107 | 63633066373136386562333938306537316664326564383261613036653135663833333538613333 108 | 66663264336434643036353835613963663837383233616436366133633036343035343039346364 109 | 35393530363739353762313138343866653265653537623534336461633633626133323631326331 110 | 65653064346462373661326134656362346530306264333463656232633433326334643837313166 111 | 66313833646130643433303164623136363061653664373366313166666437316161306666393764 112 | 35646435613139343061346631636265376664666365313139633137383332636638306565396461 113 | 65393637343031616635326536353364386431633536643766346334323436643539333235663263 114 | 39313835333033623832326164613063303734336565666164633038343334366431653838383134 115 | 63303632656462363633363565366531646166353934306239616531393761396434346131623633 116 | 62623739386461663863303839323438366434643337356639656636323664383565306365613734 117 | 64303736386131353364393633303564616330326536656364623730343234633838633630666234 118 | 38613661383532633561326462626138333631656564353534326635653564656532346166663661 119 | 33356362636637616332393531616436353430663938643061363531653866636564386230363231 120 | 32363830633165613862383632656539636139306635653234643637333233666666623838323339 121 | 35393136616662373638656531313530613139383739373834636565666533353363343265353464 122 | 66656538366661336663343966666130323236663130336462306464346536303963306432323733 123 | 32666430396530383435313831613038343536373235656631383934343732616566323332313030 124 | 32663962333865613566306462656536366362623863393430336234366636653761643337656336 125 | 30393836623238646662623365623139383235396331646133316534303462383764643662356465 126 | 32313164636266313039313139643830343033323834623865363439383766636431326238303765 127 | 66333433346431636238616363643936623664326364323666656632626435363733393238386436 128 | 34646264653231353530343237303562613964643035376231343237313838643738623134623435 129 | 61616365623662656430353636616435373936306263396431356165613462326565666663636437 130 | 38373738653564383263316339373439336632656538633536663134333334666535656562663166 131 | 31383434316238623130393833343265316665393835316561326261343633636564333031336236 132 | 64613336363635633663623238336632613336383834306366663134326664616130306664303065 133 | 61613736393530383064323334373564363935306339653534323937323833303261653531326634 134 | 61303131366139306339343866323464633565616333356238323261616263383562396264393434 135 | 61353933333263623632306566343633643835613464326334326666396463646566306666313762 136 | 35623036303266646361616130356331346466346162646636353039333332306533396439643736 137 | 32356664333536386435326632376533643062346162643031343363306230353735306633653731 138 | 64343831656366653361313839346133366133623932313130643366636663643939333938663335 139 | 66336663316131336661646435326364623131383666306563616432343261363930663338383064 140 | 32353539333730663335653163323466323363636166326361323666386639636638646432643132 141 | 61376638306462323162383961353031353263336634303463393633313866313638653930376463 142 | 62336639393333396132333835373162653932643730323063303161623232386565376139666162 143 | 39353434393266656238336634353739376435393664653339623331636330626237613962323366 144 | 30336632623439373138316432343537393332303962623239393832386664303866626666346166 145 | 32306639326263363064646332613561356233376437323030393130316235346538663934663566 146 | 39366432633833356635356531313935333331376631636136643530313861613232333633316432 147 | 32653463373334383032623034663232306631353833333735646662316531383064613939373137 148 | 66373135336334373534373434616432613632383966623466653663303363316634363434303038 149 | 64376465366539613333623564373463393238646531656436396230303734626530636335353763 150 | 33333561396363626464656465326562343037663635656435346336316266346161383463333034 151 | 62323435646161653762663334653066616565343837346634376238326533303364643663623731 152 | 64346238393032356139336635653462333830363931643538666533663464383962663262646263 153 | 36613865663333346565653563353962336336373961646461306236633735323937333237366330 154 | 33666532353065326138623434306464346430323038353131383834333532333438396439346339 155 | 36663366623661646264386362313361656432343636323964306565663630626332656635666633 156 | 32386266353937336139613665653165653536303365663334383231663466353432306365336361 157 | 30346461656630623632313130616663613062343837656236396336663365306437616435306631 158 | 37386366633331626266643163626665386132323734336635336337393135383263353235383233 159 | 64653030386231336639343733663231373233663537386235313863366333343266653635643636 160 | 30323432343237336336613237366435663564646333343566356433663337333734346366633536 161 | 30623130376635613930356535643131663139646466613665613133386330356430613065613732 162 | 62613163346263396339363630396561373462396465623137383235383933376666663736343564 163 | 35333331343462303936363030653534323664663933376364346331346161363731613430663165 164 | 63396566666362363831366231343239643863316531623133333734323931633433663230613933 165 | 62396638613962663937383936303333356361383936393932393637663933333732656235366561 166 | 30633735333766313638636634666231356364376636653932323564646430643730636362363837 167 | 34323831663133656536356438666132323138323537636131383430343836626533346337616233 168 | 38626539633564343234633239653835663633666631386364616530653061376531626235306233 169 | 33643363666661633430366562373866373438396337346530393938656466396238613735333562 170 | 31373432323733313866343035376164666237353633363766616563663733366637363365303863 171 | 63306438393961383432363637636530366133326431643535336134653563386466656539356538 172 | 33306366326332343430393137343565303832303634393936306165396664363462356638666530 173 | 61653839643233636638663761343935633664326566636230653435636532616264336562313632 174 | 62346661663733313535303037633763633163363239303362393764613161383133333233646639 175 | 38643566353732323330613231653438326461646265623833343966646634663038383635613633 176 | 35343739393038363935633830326364346165643432613764326534633663313733383461393237 177 | 30663962656333356561613466346335393731346635613330383938386531356565303533623039 178 | 39643962623765383965623838363930636462613438326134666663313865353733303264306635 179 | 66653066353366333739623332306434346131393035336632636636626261636333643063653832 180 | 30373436346539646661366265666434346232313062646462333836333462366365666131316636 181 | 33656334326166636439396164356264653466623034646663323164393633656263306336636534 182 | 63613761663137363036613231663433666464313238663536323734363661316334343135366566 183 | 31393866363864653633626462316632323439396231316565353266343664656364353261336336 184 | 31396433646234346165646233363830346666373530623736393734623166643632303137386164 185 | 61623361313231623937366561623636623339323939633666363035373365643137636464303832 186 | 35383361616437333831386631653334383935353036353034333265336639663865626463646663 187 | 62386531656663336633353366323935336436326438303235316631666532333466376137663033 188 | 66386463366133336132393331386563633931366138353561363933666137643563393665343564 189 | 64616564656566346138393731346638363864313566373631653164393033653264306561343562 190 | 31313735356631303339346464646566356335366533653734623830333330343939343230346561 191 | 61393930333366346439623439346463633063373035383865396163643039303739376132636433 192 | 33633638373034323165643361653936323464663034373232616633623230626661343337393134 193 | 38663834656435643033333632376462633166646266633363356430376464306533633334373137 194 | 36666636313466366539623735626464356530333130393535393736613030643061623534663134 195 | 31663039356363396661383536666638313738356265663631653134383737336566636537326131 196 | 36636236366635326435623661346632366163396531323235646537366463303865626164346635 197 | 63386436306430646536643764313437663266353938656361616333636562653766336364363634 198 | 31333765373236366332623931326437643439633263316633613262336236343262613861363065 199 | 39656432636439306231356531313961326633346561363666306563393661346434373737666635 200 | 37333936623438656265393064373239356365383563653965653030356161373231383464313634 201 | 61343362653062333165386436613963366230363130303232393164653766643233636663623065 202 | 38343334666566643935643163363034663734303138663634373334393531653334623165353131 203 | 32383334656638303335636333366536393436626330613238383364653537616464326566373965 204 | 32393439613039316262636131306665316337633263306536326562386561663562616238623737 205 | 37303035626434313966363661343039616238336365636637643335613532363466333064303433 206 | 39303462616436653233613639613361353662643865323333356236636663333066616230363635 207 | 35373361326663633735386333346361633739636139643237373131346434356535646363316535 208 | 36383664336162643064356364393931313835316265623531343861313239333463633133643538 209 | 36646239376133643636643932663038303839356263306466356430666530323431663539663036 210 | 33306364656163396265343266336666393230366630626465356638636431646337623436306462 211 | 66303665613562306530336336303332316466643337633937613862346639336339653430323438 212 | 33323661303663343636653738393737653434353834366237343638356564366530363232333365 213 | 65343461663061353936363333633265343564663634303230393832626466643863326364633637 214 | 34303766616631613833323764663663336531616436326135323036613931613539656433313865 215 | 33636534373834373264623264643765646566373164346133626261346365623362343261633731 216 | 62333939626264666133396235363066653064303734303463663533646363396466303366643763 217 | 34383932386534353163393863353163363165633136623433383632663361373131393133656335 218 | 36383362343534333838393534346263646333333538396262356135393130616237613732616663 219 | 64333231386431373732623165663961323738373733643837323838373433396363663165623162 220 | 30633462323931623234663363393531363364376339373461303230633835373939333364343332 221 | 64663034383030663331313331396336313565393465306161613164323066663963663432343532 222 | 35323430376534343434333435366164346163666165336330643361636230353833386362393536 223 | 64353066646434626263633166646461366431343533643735303135613738343135633966323761 224 | 64356363346165643965323637626365333736353461646534393230636565323238313061363235 225 | 35353832613634343836663132616561386636636166663734663463623631376365343865366162 226 | 35643237383134666564393931303735326337303836356434363265613638326665333433353763 227 | 34633339613165346466613837623263353934383664303964326163326463333830623439653539 228 | 37316465343038333361376336386466373132333530396232646133363836303433663831613239 229 | 31343166653732633565326631356335663461653232393935393831373936613335323362613962 230 | 34303337623434663466343166313636333037393765613931636530356261613232656163393037 231 | 3663376138623465656237396631653734363837613737393833 232 | -------------------------------------------------------------------------------- /austin/group_vars/local/vault.yaml: -------------------------------------------------------------------------------- 1 | $ANSIBLE_VAULT;1.1;AES256 2 | 35353264353432383963313538316138363638333036303035353530356437323636303236666239 3 | 6639353265313335323235313666393032643832363365340a666434396539333132613438383734 4 | 31343339616464313163636366316433323030353262346463626262383938303631386338323534 5 | 3438336264666562320a663365613432326462306338383365386339613534623638383064373531 6 | 61396433316431366231356334366338646365623932383162666262613836306161306133383835 7 | 34646262323131313832373962343632663130613362303961343933626134373532626162653831 8 | 63653835643639303238303064393665306435363233663236663433663836333265376239656266 9 | 33386630386563653361383637313832373538346630653862313233333033373233386536376437 10 | 31356366343762666439366434363233396631643163616434336665656638653039663261623461 11 | 32303335643839613664383562336130666465386363396233326233623233326137633036306131 12 | 36323664313437373430393030623932313239353233346338303932653532336366646634323166 13 | 62353738636633306565633234643863396233393534363664346235323637313237343635396630 14 | 37363231323131656664383332353361316463663964396665346130383033353631323761383065 15 | 32313562616437383230326439366261313631376237633130316262393439313538353035393233 16 | 37303461353363616435373738663465653033613964653530656133663766346566356533396436 17 | 63623961616334356361336366343539326464313938643731366131643766633838636563376466 18 | 63653734356430623839613366356636663633343365303739653137366137346261626263386530 19 | 64643234303431623637656439323230376664633630626665336263373739663966383635336564 20 | 38363632356336353730306365663030393561336362316164383066666636363839643339396335 21 | 37646630326563396436613134333130336137653836666465653830633137353734393638663936 22 | 33326362626162666564333162353433353136386630633830343333346666656332383864386135 23 | 38333864633239393665633637626662303331356439646332333365353362336562633930616331 24 | 62623761353734643937353836393534366664336362303634373063363231373263643666333037 25 | 65306337383565633734656536613138306433366632613065363633633234323962363038663766 26 | 61633534623738393038373833616535383663343961643163616165353531383535356334333032 27 | 35313636656361373237356363663562336130326666666635633538373462623231643130633937 28 | 64313034633137623866306238326532363561376561306364363734623738396437623736313635 29 | 31366365333261326233646135653532353039303230393865316637326534316365393335313938 30 | 34353337373632356134393465343236313936633532616438343866366430346462363666373866 31 | 38393134366537333363353137633436633835663832383764613662643638363063646632336265 32 | 64353535646632393838323031366635643265356233663130303038363566316166396432353431 33 | 30663162626136636364366438353436343364326663613034366339326138333134333663623232 34 | 34313366353838633062656365623761343061643734333030646134613765383337393538366162 35 | 33326261613935613665326133363461636131316538313861386533336163636464373033383863 36 | 65373833616634353632663035366565393064623466333064323762346238643363333636323734 37 | 62383265363738363130393662616463366262373935633332636130306331356366343030626561 38 | 35643138383934383535613838306162633639313865663166653065363334333962666435333239 39 | 31326363623334653831306536373065653234343333393533393736313334613365623035316362 40 | 62663164663162393363656233373564366436353537373637666438306664356434616466646636 41 | 61313162613034376639376133303732616164653934376231616236366538373462383230643265 42 | 66653865303038313431326535633833383166613038633830333462653365613138366439353463 43 | 30303465313763366432343330336630386565643362643063363437663032636362663061643531 44 | 32343263303733313461663839386536616662313930346533636137623439313036646332336263 45 | 37386634646564316162386564323964633431316664656331646136623832343836616534383537 46 | 61393334333233373966373732663737643066666333656264613162353961316534333836653165 47 | 36633238656264666130396530333364396537313231646332613533643836653538383137323365 48 | 37393363326432376463383133363066393064656235313834663439333764303433616335313965 49 | 64323539663133303732623262623365356635346639653361313761326264373666393966363938 50 | 36613638623665656362393030613166666233326161333239323532616462373162666131613836 51 | 37633035373930633762373362323363666666643237366466336364643764613137386561653665 52 | 64623737303463633864323739633662323761306435313666393863663464616564313765386464 53 | 62363964313736373736613966623731346265323963656536373936373631373865303732366631 54 | 64383432373165323639373738313836656333623864333365363130343035646633306437333533 55 | 37306165323763336636623137303435626333356366376234663833306336323739616338343961 56 | 33663662306335383933343366633330363630353961653232636265346536653432666530623831 57 | 61346638613733646638633433363334666565613132346633636137393430663336346464643266 58 | 64303164313064663661336230343837323633616663303865613237616230373064663730663662 59 | 35353636323030326365663039346230653933353961663737316432353464363762376331616265 60 | 34316134303163613738643335653935343334346336326339656331663538343765343032666233 61 | 34623066393635633930386331366539323438303464633166643636376236336361316234633536 62 | 39633636333133663363616537653365653530353732633565653936303562303133643130396637 63 | 39666561376232343231646332613161393035656333623864623730343061356533313963363763 64 | 38396632353436636432386336353661386636626631316666663131656265653032303635343039 65 | 38393237663634653638646339363831646232346433336634303063613233393634383539393063 66 | 64656434336564303537323561663439626139643431396365386439393236386261393862653632 67 | 62333636633563653130336235363835643563393966386137363566303933653564616235343436 68 | 30333763316666356233363763336331353933353238376233343864313335376333663261633937 69 | 38396138663239643231386636336365353633316332396539626363336236363765303962326434 70 | 39343432633537326338313663633134343965353039323163623836353064643133373738333434 71 | 61623536393364623635336230396136356230643638316231623866346231386230346430353463 72 | 64346437613334633062656535626533333065373838313166666232393931386434383837356262 73 | 36636464653738323561623664663561643065393235623438623632353133343263373834303630 74 | 65323362663663653364646234333565383461326265666362323262393735383330653839323637 75 | 32313932333431623163303833393864653632613662383333363064333564663261646636636333 76 | 32386435363039353032623030323239336166623633613831373836326138663533393862353465 77 | 39636233653665373863666233303335663562323432343535383665363730336365643662386432 78 | 36366639343632623637353031306462343564323834336139363934343662643561666636613666 79 | 33623831343264376164353231306136303863626537366330663531376466653132353633633130 80 | 36393436663466336434636336643531633837663738303965646239656231623631313733363433 81 | 34643331313464353062396435636562663130336437316362643165636139356338323531363630 82 | 32663439636464366338323864623064343661396638343031336330363866663937326633636139 83 | 38653630386436633836366539663438626165396261343664316564346138643264393561666562 84 | 63633834663163623766336264626130386336336335303364386636663566666639363832613364 85 | 64356239643930383532303437396535663633313732633338646439353466353664343561353566 86 | 64636232396461343762343231613438636130653661643331303864383763626537363331623362 87 | 62623832396137393235663332386139613138663836366464323966333237653963356564623137 88 | 37626630373233343466376638333965656130616564313337353961636632353164353635346631 89 | 33343431636237353136346238373766346133303164363435666466363835613033303535363864 90 | 33346236333130333062316337613964663336656332616237643932383063326637663736636239 91 | 34363064363232386637636337336437303461626339316232363232336233663066333361303333 92 | 35353563353066343930363762643331616536666465346236663462353930386130643139643937 93 | 63643434316132356464656361663464393532313561313533323434353331323534636434353531 94 | 35363461613261316661346462366630383730646562303062373262313763663439353561663631 95 | 62396330633863356563323333323439306139303035626462656334336362343965373038656164 96 | 39616333623963643335313463656539363834663463393434343265336235313930653833653933 97 | 63393466626334353163306238633863353431376236313336343735346139663333626134653333 98 | 30343832666661366333376433616531386561313237336532646538386134356661383161616664 99 | 66386235663361626664633833373263353534356161363432656365346636306533363632306462 100 | 32646132386635656533653563386136633133356230396132666234303931336338383062353333 101 | 35633938613864386264333535373635303363306534333035613236656432643363343362346334 102 | 64613130313535306565333661323663326233613561623163656438366535386534363238663733 103 | 39623734626165353435356564623366306466653861366461363233316230613066336537613138 104 | 30326432343439383363636134613864346331643835616238636531613338666365636336393761 105 | 38363363393831393934356663353338636164633939616464366534383236376137326533373733 106 | 62623431623761316265646334333263313036336438366430333230316436656435663631643261 107 | 30376365633030356332376235633230376633383962356637633732613763373732353939663165 108 | 33343165383831363331313261643733643232346562363566363238356530656239306231306137 109 | 64353034326264353335663462323133343338313864656635366539633763353366363762393764 110 | 38333762336439323963636633383230346633353336333532666565633136646561316235313461 111 | 62626134336665396466656336653463366531306233313535386231326239643962636563363532 112 | 32616361656239626134663661643539636233633365616639373866656435646431386263376334 113 | 39363033333536653038633965356534313937336239643239343932646664613338623964373533 114 | 35343636356565663935623033646134613336343766363761646637396235383335633833353939 115 | 30336331383638373561356433313463333533666230613436666631386434363866386133663031 116 | 33633861396231306630333662376138616462643734353838646636303535323462383134633637 117 | 30376630353864333132353162323564623864363262313232353236366138653233343831646635 118 | 61316533643462633164353234333364313930373665666361663539653539373763623035616663 119 | 37643534653936616366316531613631303539366233346533353837616564313164353436373336 120 | 64343563313432636236663430616536633835366165326463613363373839663634333164383263 121 | 66646366666137353662363838326361373064653636623161323435653339643834333032333865 122 | 38613933626464346163393364353066663535656366666163383034653536346137386363316531 123 | 62383031303962323961303233356463313864636630363563333265613831303463343635366639 124 | 38663036316133383237353039636131643931366566613531363838306537316138313666336633 125 | 63383333653532653638336535356638373631353031316539353633333931663536393762333966 126 | 66363635363565333434313435643235373464616338633330303064616262333438303065383130 127 | 37393930373165306430366136653136343436386339303135666364316336336432383530313137 128 | 34626232633862356433303866386636386637613165376638343861383435336636383933383038 129 | 35356534336430666139616238663437333736363431303030653039343663336665313333336333 130 | 35666637346133636463663165336131353135393765353837353833633464633132323439303837 131 | 34353166333466356330383165303133333833333435646566303238326438303466353038616330 132 | 31363535333761366632646363626562666634656164393264393130396538626635313237373030 133 | 31316436343861326566333338643237303363623139656536616533323730626535376534393532 134 | 34376566666435613162633439373466303235353165326231376431313765363937343733333631 135 | 36336264353730303233653532333466373530613461336130386436313330333735643465353331 136 | 36646535383934353561653631353563653237356365626132343837656165646461316232336631 137 | 66306238623932613364356531323930346133366565333362663938663063653463626662376633 138 | 62303535343231626630373661633464636437313331636464373735353763313761366134623231 139 | 37383133353330643566663730336132336566616230623935356230323730353762633834633434 140 | 64363532353234356432346263373165336561643938356136313762646263346261376133326163 141 | 64653939343365656165613536623239306533616536623663353837326537363737306136633165 142 | 31666134653462373937646263343766326665613236613431316163316364336632346537353438 143 | 65356330653830646233306266306665383930373861323139306231616631343765333034353136 144 | 35613538353563346236623833366337613134636238386565646433373830653634613561623163 145 | 36643464373863616664346338653238363433373833643132656236333435633537636439656436 146 | 66323962313634393030386639623830663631613230333537653263353339366664626533373061 147 | 66383536373130636266363863313939626135646235383832323933656635313663633232646265 148 | 38663835666361616337393161613937333863613938343538343564333232646230303034356538 149 | 38656462393436623038333535376434616239343537636562363538323937323863316636396333 150 | 31306165643466613963383831646464363334323665396662616536656633653964646137336431 151 | 66626664646561623633393733663138656461643563356239303836393666366238656563356563 152 | 62636164383862623961653839363936626535653534656530393536626336396330643531333062 153 | 37653234663266353836316533653039643131623636666332313134353164376161613136393132 154 | 66663432383862383730626235383431653932363136666636343165306163376432636164373334 155 | 30383037616634643463653831353961653030363435346632363130326364303066363061383139 156 | 32333935366363666538643632396435333732376364373037636438613066653463353066353562 157 | 37666532633536353661373631313735623431663032373535383633306534363361356237313965 158 | 35386531333831656362313535313439373331326336333636393466383662306139343233623232 159 | 32343532386538613339303538356238643334633336393764373232666238383439333437326331 160 | 64383335333765616434393733323162363934356630363531376536373132376263613562363063 161 | 33323132306136383531643366336131386265353766313335396364656463633266366263393137 162 | 63373139613434383337313739613161383764313031396530653931623165653832393434313433 163 | 66333038373836336538396532313866343634626264653565336533373732613237623863643765 164 | 36323761356431326632643364616565333466626634313261623431643833366465613733383235 165 | 61373936393765643062363932666161356436306534633035323762383062333833663431333536 166 | 38316239613431366531363133366364323639323639613965643561326133323530303565623832 167 | 39623963383261623234663566323665343434306436303936623065653337313438303064393933 168 | 32353131383737326431636632643835313231306161663432323266623438663239306234353764 169 | 35623765613030623132376463366434656231376364303637393063643366626264313430373535 170 | 33336164393135663931333164643466383836313966663461653436663337303435383065613765 171 | 31366235636532633938393133383833316239323564633139373530333661626565636238666538 172 | 38336337363936616231333038633163613562396665326661663562623133383933353737306161 173 | 64643533303739393438376265313339316232643238363631643033663137343935346130333362 174 | 61313930386330646131653332366564366435633937363363343261633966636338613265316166 175 | 32373866356162636561623934343633313763333561366430643335616339373232363262376333 176 | 30356531303630313764373830396639373666663936623430373237346138666565306262613264 177 | 63373639323135303438663430373265326364323165393463366164653837376663303330303064 178 | 34353836343562353561663034653565316261613735383565346666626230626138653939333861 179 | 30623032666336336230393962353431316162613662333137636534393439353766633037626461 180 | 64383361356537363834343236383932313933383161626131373464306135356535663664363566 181 | 31303863323836646339363862623164396637303636666639363563646230356334353133306635 182 | 30346563623433626663616132323564353261626639643930626237386166613163303763376266 183 | 30633539653036323465613130653033366633373535393538336630306237313363363636323431 184 | 31333336303231316230353534363633313034333033353030636464613265326666326434336466 185 | 31316430363562376263346365633464343135643865343630396332303666383935613535323838 186 | 36326636613832333032343164653562613265356434666362366430313038633862653766313864 187 | 61636164323465643533636230306539386535336138323236306564333461343264343963376337 188 | 39626566396162346534353836633063323064363539396366343061613636373536373938633738 189 | 30616665376365643037656363666231393835393565363836616363623838386364353133353436 190 | 35356130396566316166333165336264303761333133383939313439336335313439326137633238 191 | 62343237663236613434343034333036663762646561653432663134396334653130346232313066 192 | 34666530376636653332366333353432366562373930613437393932363638643137303331383931 193 | 35333035313936353931323439323563343964366436646466643939663265303835373430373539 194 | 33663839353364356461326131663834383434373736383938356638316231633531633533363437 195 | 36376136616230663035346661396536666166303635616435313832363864373035376233303464 196 | 38326365323366666130353731396365343530366334396432313237316566333563343762346332 197 | 63656562316333306135363662636530356131343132386438303265376662656565366537346435 198 | 33626534393766313132313163633164623863353332646533653830636130653362306361323262 199 | 33303664396338306666306364653130646433333765653539636564643233343164363961356361 200 | 39616366616461336535643137613037303438326566303863613733396664653936633565366366 201 | 63623464356130316666323065363138636333626237353833343765656234653264643937343664 202 | 65613432636631313239663765663366306335386337303538393966393933663764643332653432 203 | 63656131643635613962326337373430346338363034393838323131356130383330326263663763 204 | 32656430616331373238333465666665343535363866303761633334346239616237626538616235 205 | 36626564396163393238333862616266316664333764326561326662396536326633633032663662 206 | 34303031653733623930303630326438323133353061623734663736633133643135303761653866 207 | 32636435303438316634663365353361663765626138626136663064383065313636663864643933 208 | 62353535376232303334363266326663303036656662616237373662633565366639343137383635 209 | 65353961323839393938383366396433326335316330666266313064366331393965373362653730 210 | 33336162373535666366646665626331333561333664356233323463613539323231666635323364 211 | 35636137633230323138363661626533346438373563656431333931393133663366653561343661 212 | 33326136336238356262613937396565393966316232656163336636633561626435303432646562 213 | 63343332303536383866613637343731353061306238383436623666303335313939636165383262 214 | 33396434343637326335663163346334656439336433656634393439636663666631623230323562 215 | 35613231356164633431313331393133313163326365336137386139666562663730636533393761 216 | 31363331353430356663623962316330346237373730323933363132336465323961366637386333 217 | 34643634643364353030336639333739613635613333646164623831363266386466346462643463 218 | 33393262313533643230633765306362646231363764376233373231323338383664306233326630 219 | 33383234663461336435623862303432656634353963383031353138306132313832626139313831 220 | 62643837623038343538663865323134383135306366313738373731616261303639616533356535 221 | 35323933646536623132313834383431323865306139386438623363336662383361643238613937 222 | 63353230613665383662393631343739333666373234343139373634396665393766393234343430 223 | 36326136663831363736343335633035313065363832363932613764613938646230646165666136 224 | 39613366353932336263366464313532303866366337383831363737303264613731383531326132 225 | 37346334326139633262346262616164616565643439333032353436366633313137646565333361 226 | 39316139383463363638626337653939383662333264353539646266643163343766656366633166 227 | 63643634616531613735313230383033356266383862633733316336643237313036636433343233 228 | 34356538643864633339616562383232353863373335373234303462333466363438313763393033 229 | 62323863633930323736353737616434343637316435313062666264643232633436366533323230 230 | 61666362386632373035613231396537386532383834653864303535656162366631643338346666 231 | 6233353935623031336266393933666264393961336231356632 232 | --------------------------------------------------------------------------------