├── roles ├── proxy │ ├── templates │ │ ├── ssl.crt.j2 │ │ ├── ssl.key.j2 │ │ ├── Dockerfile.j2 │ │ ├── docker-compose.yml │ │ └── nginx.conf.j2 │ ├── defaults │ │ └── main.yml │ └── tasks │ │ └── main.yml ├── docker │ ├── templates │ │ ├── ca.pem.j2 │ │ ├── cert.pem.j2 │ │ ├── key.pem.j2 │ │ └── docker_config │ ├── defaults │ │ └── main.yml │ └── tasks │ │ └── main.yml ├── restuser │ ├── vars │ │ └── RedHat.yml │ ├── tasks │ │ ├── login-defs.yml │ │ ├── install.yml │ │ └── main.yml │ └── templates │ │ └── restuser.service.j2 ├── common │ ├── handlers │ │ └── main.yml │ ├── templates │ │ └── other-pubkeys.j2 │ ├── tasks │ │ ├── python.yml │ │ ├── main.yml │ │ └── ssh.yml │ ├── defaults │ │ └── main.yml │ └── files │ │ └── sshd_config ├── nfs_server │ ├── defaults │ │ └── main.yml │ ├── tasks │ │ ├── main.yml │ │ ├── bindvolume.yml │ │ └── server.yml │ └── templates │ │ └── exports ├── nfs_server_dedicated │ ├── defaults │ │ └── main.yml │ ├── tasks │ │ ├── main.yml │ │ └── server.yml │ └── templates │ │ └── exports ├── hub │ ├── templates │ │ ├── cluster.j2 │ │ ├── userlist.j2 │ │ └── docker-compose.yml │ ├── tasks │ │ ├── swarm.yml │ │ ├── main.yml │ │ └── jupyterhub.yml │ ├── files │ │ └── get_user_id.sh │ └── defaults │ │ └── main.yml ├── rackspace │ ├── default │ │ └── main.yml │ ├── tasks │ │ └── main.yml │ └── files │ │ ├── backup.sh │ │ └── monitoring.sh └── nfs_client │ ├── defaults │ └── main.yml │ └── tasks │ └── main.yml ├── group_vars ├── jupyterhub_nfs ├── jupyterhub_nodes └── jupyterhub_host ├── users.vault.yml.example ├── script ├── deploy ├── assemble_certs └── launch.py ├── deploy-part1.yml ├── host_vars ├── example ├── jupyterhub_host ├── jupyterhub_node1 ├── jupyterhub_node2 └── proxy_server ├── inventory ├── deploy-part2.yml ├── inventory.example ├── users.vault.yml ├── reboot.yml ├── deploy.yml ├── .gitignore ├── secrets.vault.yml.example ├── LICENSE.txt ├── INSTALL.md ├── README.md └── secrets.vault.yml /roles/proxy/templates/ssl.crt.j2: -------------------------------------------------------------------------------- 1 | {{ ssl_cert }} 2 | -------------------------------------------------------------------------------- /roles/proxy/templates/ssl.key.j2: -------------------------------------------------------------------------------- 1 | {{ ssl_key }} 2 | -------------------------------------------------------------------------------- /group_vars/jupyterhub_nfs: -------------------------------------------------------------------------------- 1 | --- 2 | rackspace_backup: yes -------------------------------------------------------------------------------- /roles/docker/templates/ca.pem.j2: -------------------------------------------------------------------------------- 1 | {{ docker_ca_cert }} 2 | -------------------------------------------------------------------------------- /roles/docker/templates/cert.pem.j2: -------------------------------------------------------------------------------- 1 | {{ docker_tls_cert }} 2 | -------------------------------------------------------------------------------- /roles/docker/templates/key.pem.j2: -------------------------------------------------------------------------------- 1 | {{ docker_tls_key }} 2 | 3 | -------------------------------------------------------------------------------- /roles/restuser/vars/RedHat.yml: -------------------------------------------------------------------------------- 1 | --- 2 | python_interpreter: "/usr/bin/python3" 3 | -------------------------------------------------------------------------------- /group_vars/jupyterhub_nodes: -------------------------------------------------------------------------------- 1 | --- 2 | docker_server_tls: yes 3 | docker_client_tls: yes -------------------------------------------------------------------------------- /roles/proxy/templates/Dockerfile.j2: -------------------------------------------------------------------------------- 1 | FROM nginx 2 | ADD nginx.conf /etc/nginx/nginx.conf -------------------------------------------------------------------------------- /roles/common/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: restart sshd 3 | service: name=ssh state=restarted 4 | -------------------------------------------------------------------------------- /roles/nfs_server/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Path to the NFS volume on the host 3 | nfspath: '' 4 | -------------------------------------------------------------------------------- /group_vars/jupyterhub_host: -------------------------------------------------------------------------------- 1 | --- 2 | docker_server_tls: no 3 | docker_client_tls: yes 4 | rackspace_backup: yes -------------------------------------------------------------------------------- /roles/common/templates/other-pubkeys.j2: -------------------------------------------------------------------------------- 1 | {% for key in other_ssh_keys %} 2 | {{ key }} 3 | {% endfor %} 4 | -------------------------------------------------------------------------------- /roles/nfs_server_dedicated/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Path to the NFS volume on the host 3 | nfspath: '' 4 | -------------------------------------------------------------------------------- /roles/hub/templates/cluster.j2: -------------------------------------------------------------------------------- 1 | {% for host in groups['jupyterhub_nodes'] %} 2 | {{ hostvars[host]['servicenet_ip'] }}:{{ docker_server_port }} 3 | {% endfor %} 4 | -------------------------------------------------------------------------------- /roles/hub/templates/userlist.j2: -------------------------------------------------------------------------------- 1 | {% for user in jupyterhub_admins %} 2 | {{ user }} admin 3 | {% endfor %} 4 | {% for user in jupyterhub_users %} 5 | {{ user }} 6 | {% endfor %} 7 | -------------------------------------------------------------------------------- /roles/nfs_server_dedicated/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - fail: msg="nfspath is not defined" 3 | when: nfspath == '' 4 | 5 | - name: setup NFS server 6 | import_tasks: server.yml 7 | -------------------------------------------------------------------------------- /roles/hub/tasks/swarm.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: create the /root/.swarm directory 3 | file: path=/root/.swarm state=directory 4 | 5 | - name: create the cluster list 6 | template: src=cluster.j2 dest=/root/.swarm/cluster 7 | -------------------------------------------------------------------------------- /roles/hub/files/get_user_id.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | 3 | user=$1 4 | response=$(echo -e "POST /$user HTTP/1.0\r\n" | nc -U /var/run/restuser.sock) 5 | uid=$(echo "$response" | grep -o "\"uid\":\ [0-9]\+" | cut -c 8-) 6 | echo -n $uid 7 | -------------------------------------------------------------------------------- /roles/nfs_server/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - fail: msg="nfspath is not defined" 3 | when: nfspath == '' 4 | 5 | - name: setup NFS server 6 | import_tasks: server.yml 7 | - name: bind NFS volume to local 8 | import_tasks: bindvolume.yml 9 | -------------------------------------------------------------------------------- /roles/common/tasks/python.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: python 3 | apt: name=python state=latest 4 | 5 | - name: pip install script 6 | get_url: dest=/tmp/get_pip.py url=https://bootstrap.pypa.io/get-pip.py 7 | 8 | - name: pip 9 | command: python /tmp/get_pip.py creates=/usr/local/bin/pip 10 | -------------------------------------------------------------------------------- /roles/nfs_server/tasks/bindvolume.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: bind exchange nfs dir 3 | mount: src={{ nfspath }}/exchange name=/exchange fstype=none opts=bind state=mounted 4 | 5 | - name: bind jupyter nfs dir 6 | mount: src={{ nfspath }}/jupyter name=/jupyter fstype=none opts=bind state=mounted 7 | 8 | -------------------------------------------------------------------------------- /roles/restuser/tasks/login-defs.yml: -------------------------------------------------------------------------------- 1 | - lineinfile: 2 | dest: /etc/login.defs 3 | regexp: '^(UMASK\s+).+$' 4 | line: '\g<1>022' 5 | backrefs: yes 6 | 7 | - lineinfile: 8 | dest: /etc/login.defs 9 | regexp: '^(HOME_MODE\s+).+$' 10 | line: '\g<1>0755' 11 | backrefs: yes 12 | -------------------------------------------------------------------------------- /roles/rackspace/default/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Never actually include your Rackspace username and API ey here -- they should 3 | # be overwritten in an encrypted vault file. 4 | rackspace_username: '' 5 | rackspace_api_key: '' 6 | 7 | # Default is to not backup, but this can be overwritten. 8 | rackspace_backup: no 9 | -------------------------------------------------------------------------------- /roles/restuser/tasks/install.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: python 3 | yum: name=python3,python3-pip,git state=latest 4 | 5 | - name: tornado 6 | pip: name=tornado state=latest executable=/usr/bin/pip3 7 | 8 | - name: install restuser 9 | git: repo=https://github.com/minrk/restuser.git dest=/srv/restuser 10 | 11 | -------------------------------------------------------------------------------- /roles/nfs_client/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Inventory file hostname of the NFS server (note this should be an Ansible 3 | # alias, not the real hostname) 4 | nfsserver: '' 5 | # Options to mount NFS with 6 | nfsoptions: rw,_netdev,auto 7 | # Where the exchange directory should be mounted on the client 8 | exchange: '' 9 | -------------------------------------------------------------------------------- /roles/common/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: install vim for ops sanity 3 | apt: update_cache=yes cache_valid_time=600 name=vim state=latest 4 | 5 | - name: YOLO system upgrade 6 | apt: upgrade=safe 7 | 8 | - name: install git 9 | apt: name=git state=latest 10 | sudo: yes 11 | 12 | - include: ssh.yml 13 | - include: python.yml 14 | -------------------------------------------------------------------------------- /roles/common/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # You can give admins SSH access either by specifying their GitHub usernames 3 | # here, which will download all their keys from GitHub (i.e., from 4 | # https://github.com/username.keys). 5 | github_usernames: [] 6 | 7 | # If you don't want to give all of those keys SSH access, you can specify 8 | # individual keys here: 9 | other_ssh_keys: [] -------------------------------------------------------------------------------- /roles/docker/templates/docker_config: -------------------------------------------------------------------------------- 1 | {% if docker_server_tls %} 2 | DOCKER_OPTS="--icc=false --tlsverify --tlscacert={{ docker_tls_path }}/ca.pem --tlscert={{ docker_tls_path }}/cert.pem --tlskey={{ docker_tls_path }}/key.pem -H=0.0.0.0:{{ docker_server_port }} -H=unix:///var/run/docker.sock" 3 | {% else %} 4 | DOCKER_OPTS="--icc=false -H=unix:///var/run/docker.sock" 5 | {% endif %} 6 | -------------------------------------------------------------------------------- /roles/nfs_server/templates/exports: -------------------------------------------------------------------------------- 1 | {% for host in groups[nfsnodes_group] %} 2 | {{ nfspath }} {{ hostvars[host]['servicenet_ip'] }}(rw,fsid=0,no_root_squash,no_subtree_check,sync) 3 | {{ nfspath }}/exchange {{ hostvars[host]['servicenet_ip'] }}(rw,no_root_squash,no_subtree_check,sync) 4 | {{ nfspath }}/jupyter {{ hostvars[host]['servicenet_ip'] }}(rw,no_root_squash,no_subtree_check,sync) 5 | {% endfor %} 6 | -------------------------------------------------------------------------------- /roles/restuser/templates/restuser.service.j2: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=restuser 3 | After=network.target 4 | 5 | [Service] 6 | ExecStart={{ python_interpreter }} restuser.py --socket=/var/run/restuser.sock --skeldir=/etc/skel --homedir=/jupyter/users/USERNAME 7 | ExecStop=/bin/kill -INT ${MAINPID} 8 | Restart=always 9 | WorkingDirectory=/srv/restuser/ 10 | 11 | [Install] 12 | WantedBy=multi-user.target 13 | -------------------------------------------------------------------------------- /roles/nfs_server_dedicated/templates/exports: -------------------------------------------------------------------------------- 1 | {% for host in groups[nfsnodes_group] %} 2 | {{ nfspath }} {{ hostvars[host]['servicenet_ip'] }}(rw,fsid=0,no_root_squash,no_subtree_check,sync) 3 | {{ nfspath }}/exchange {{ hostvars[host]['servicenet_ip'] }}(rw,no_root_squash,no_subtree_check,sync) 4 | {{ nfspath }}/jupyter {{ hostvars[host]['servicenet_ip'] }}(rw,no_root_squash,no_subtree_check,sync) 5 | {% endfor %} 6 | -------------------------------------------------------------------------------- /users.vault.yml.example: -------------------------------------------------------------------------------- 1 | --- 2 | # Rename this file to users.vault.yml and edit it to include usernames for your 3 | # own admins and users. 4 | # 5 | # Note that after you do this, you should encrypt the file with ansible vault 6 | # so that the usernames are not public: 7 | # 8 | # ansible-vault encrypt users.vault.yml 9 | # 10 | jupyterhub_admins: 11 | - jhamrick 12 | jupyterhub_users: 13 | - minrk 14 | - rgbkrk -------------------------------------------------------------------------------- /script/deploy: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ROOT=$(dirname $0)/.. 4 | 5 | suffix="$1" # either "-part1" or "-part2" 6 | shift 7 | 8 | if [ ! -e ${ROOT}//inventory ]; then 9 | echo "Please create an inventory file with your hosts." 10 | echo " cp inventory.example inventory" 11 | exit 1 12 | fi 13 | 14 | VAULT_ARG="--vault-password-file ${ROOT}/vault-password" 15 | exec ansible-playbook ${ROOT}/deploy${suffix}.yml -i ${ROOT}/inventory ${VAULT_ARG} $@ 16 | -------------------------------------------------------------------------------- /deploy-part1.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - hosts: 3 | # - proxy # (turn off root_nginx_1, replaced by root_nginx_3) 4 | - jupyterhub_nfs 5 | - jupyterhub_host 6 | - jupyterhub_nodes 7 | vars_files: 8 | - 'secrets.vault.yml' 9 | roles: 10 | - common 11 | 12 | - hosts: 13 | # - proxy # (turn off root_nginx_1, replaced by root_nginx_3) 14 | - jupyterhub_host 15 | - jupyterhub_nodes 16 | vars_files: 17 | - 'secrets.vault.yml' 18 | roles: 19 | - docker 20 | -------------------------------------------------------------------------------- /host_vars/example: -------------------------------------------------------------------------------- 1 | --- 2 | # There should be one file per host, named with the inventory hostname. The 3 | # variables defined in that file will be used for that host. For example, to 4 | # configure the docker TLS certificates, you would want to have defined: 5 | 6 | docker_ca_cert: '' 7 | docker_tls_cert: '' 8 | docker_tls_key: '' 9 | 10 | # For the proxy server, you would want to have defined: 11 | 12 | ssl_cert: '' 13 | ssl_key: '' 14 | ssl_cert_path: '' 15 | ssl_key_path: '' 16 | -------------------------------------------------------------------------------- /roles/proxy/templates/docker-compose.yml: -------------------------------------------------------------------------------- 1 | staticfiles: 2 | image: triggers/systemuser 3 | command: echo "#data-container-hack" 4 | volumes: 5 | - "{{ notebook_static_files }}" 6 | 7 | nginx: 8 | build: /srv/nginx 9 | restart: always 10 | ports: 11 | - 80:80 12 | - 443:443 13 | volumes: 14 | - "{{ ssl_key_path }}:{{ ssl_key_path }}:ro" 15 | - "{{ ssl_cert_path }}:{{ ssl_cert_path }}:ro" 16 | volumes_from: 17 | - staticfiles 18 | -------------------------------------------------------------------------------- /roles/rackspace/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - fail: msg="rackspace_username is not defined" 3 | when: rackspace_username == '' 4 | - fail: msg="rackspace_api_key is not defined" 5 | when: rackspace_api_key == '' 6 | 7 | - name: install rackspace monitoring agent 8 | script: monitoring.sh {{ rackspace_username }} {{ rackspace_api_key }} 9 | 10 | - name: install rackspace backup agent 11 | script: backup.sh {{ rackspace_username }} {{ rackspace_api_key }} 12 | when: rackspace_backup 13 | -------------------------------------------------------------------------------- /roles/rackspace/files/backup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh -e 2 | 3 | if [ -z $(which driveclient) ]; then 4 | apt-get update 5 | apt-get install python-apt 6 | cd /tmp 7 | wget 'http://agentrepo.drivesrvr.com/debian/cloudbackup-updater-latest.deb' 8 | dpkg -i /tmp/cloudbackup-updater-latest.deb || true 9 | apt-get install -f 10 | cloudbackup-updater -v 11 | /usr/local/bin/driveclient --configure -u $1 -k $2 12 | service driveclient start 13 | else 14 | service driveclient restart 15 | fi 16 | -------------------------------------------------------------------------------- /inventory: -------------------------------------------------------------------------------- 1 | [jupyterhub_host] 2 | hub ansible_ssh_user=root ansible_ssh_host=192.168.11.88 servicenet_ip=192.168.11.88 3 | 4 | [jupyterhub_nodes] 5 | node1 ansible_ssh_user=root ansible_ssh_host=192.168.11.1 fqdn=node1 servicenet_ip=192.168.11.1 6 | node2 ansible_ssh_user=root ansible_ssh_host=192.168.11.2 fqdn=node2 servicenet_ip=192.168.11.2 7 | 8 | [jupyterhub_nfs] 9 | hub 10 | 11 | ## Not deploying root_nginx_1 container. It is now completely replaced by 12 | ## root_nginx_3 which is deployed by bashscripts. So commenting out the next two lines: 13 | # [proxy] 14 | # hub 15 | 16 | [nfs_clients] 17 | node1 18 | node2 19 | -------------------------------------------------------------------------------- /roles/rackspace/files/monitoring.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh -e 2 | 3 | if [ -z $(which rackspace-monitoring-agent) ]; then 4 | sh -c 'echo "deb http://stable.packages.cloudmonitoring.rackspace.com/ubuntu-15.04-x86_64 cloudmonitoring main" > /etc/apt/sources.list.d/rackspace-monitoring-agent.list' 5 | curl https://monitoring.api.rackspacecloud.com/pki/agent/linux.asc | apt-key add - 6 | apt-get update 7 | apt-get install rackspace-monitoring-agent 8 | rackspace-monitoring-agent --setup --username $1 --apikey $2 9 | service rackspace-monitoring-agent start 10 | else 11 | service rackspace-monitoring-agent restart 12 | fi 13 | -------------------------------------------------------------------------------- /deploy-part2.yml: -------------------------------------------------------------------------------- 1 | ## Not deploying root_nginx_1 container. It is now completely replaced by 2 | ## root_nginx_3 which is deployed by bashscripts. So commenting out the next section: 3 | 4 | #- hosts: proxy 5 | # vars_files: 6 | # - 'secrets.vault.yml' 7 | # roles: 8 | # - proxy 9 | 10 | - hosts: jupyterhub_nfs 11 | vars_files: 12 | - 'secrets.vault.yml' 13 | roles: 14 | - nfs_server 15 | 16 | - hosts: nfs_clients 17 | vars_files: 18 | - 'secrets.vault.yml' 19 | roles: 20 | - nfs_client 21 | 22 | - hosts: jupyterhub_host 23 | vars_files: 24 | - 'secrets.vault.yml' 25 | - 'users.vault.yml' 26 | roles: 27 | - restuser 28 | - hub 29 | -------------------------------------------------------------------------------- /roles/proxy/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # These are the paths to the SSL certificate and key. This should be overwritten 3 | # in an encrypted vault file. 4 | ssl_cert_path: '/tmp/proxycert' 5 | ssl_key_path: '/tmp/proxykey' 6 | jupyterhub_host: 192.168.11.88 7 | 8 | # These are the contents of the SSL certificate and key themselves. These should 9 | # also be overwritten in an encrypted vault file. 10 | 11 | # The keys get wrapped when pasted here....copy the cert and key to 12 | # the above files manually after ansible finishes 13 | ssl_cert: '' 14 | ssl_key: '' 15 | 16 | # This is the path to the static files for the singleuser notebook server 17 | notebook_static_files: /usr/local/lib/python3.4/dist-packages/notebook/static/ 18 | -------------------------------------------------------------------------------- /roles/nfs_client/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - fail: msg="nfsserver is not defined" 3 | when: nfsserver == '' 4 | - fail: msg="exchange is not defined" 5 | when: exchange == '' 6 | 7 | - name: install nfs client 8 | yum: name=nfs-utils state=present 9 | 10 | - name: unmount exchange nsf dir 11 | shell: umount {{ exchange }} || true 12 | 13 | - name: unmount jupyter nsf dir 14 | shell: umount /jupyter || true 15 | 16 | - name: mount exchange nfs dir 17 | mount: path={{ exchange }} src={{ hostvars[nfsserver]['servicenet_ip'] }}:/exchange fstype=nfs opts={{ nfsoptions }} state=mounted 18 | 19 | - name: mount jupyter nfs dir 20 | mount: path=/jupyter src={{ hostvars[nfsserver]['servicenet_ip'] }}:/jupyter fstype=nfs opts={{ nfsoptions }} state=mounted 21 | -------------------------------------------------------------------------------- /inventory.example: -------------------------------------------------------------------------------- 1 | [jupyterhub_host] 2 | jupyterhub_host ansible_ssh_user=root ansible_ssh_host= servicenet_ip= 3 | 4 | [jupyterhub_host] 5 | jupyterhub_nfs ansible_ssh_user=root ansible_ssh_host= servicenet_ip= 6 | 7 | [jupyterhub_nodes] 8 | jupyterhub_node1 ansible_ssh_user=root ansible_ssh_host= fqdn= servicenet_ip= 9 | jupyterhub_node2 ansible_ssh_user=root ansible_ssh_host= fqdn= servicenet_ip= 10 | 11 | [proxy] 12 | proxy_server ansible_ssh_user=root ansible_ssh_host= jupyterhub_host= servicenet_ip= 13 | 14 | [nfs_clients] 15 | jupyterhub_host 16 | jupyterhub_node1 17 | jupyterhub_node2 -------------------------------------------------------------------------------- /roles/restuser/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - include_vars: "{{ item }}" 3 | with_first_found: 4 | - "{{ ansible_os_family }}-{{ ansible_distribution_major_version}}.yml" 5 | - "{{ ansible_os_family }}.yml" 6 | 7 | - include_tasks: login-defs.yml 8 | 9 | - yum: name=epel-release 10 | 11 | - name: install restuser 12 | include_tasks: "{{ item }}" 13 | with_first_found: 14 | - "install-{{ ansible_os_family }}-{{ ansible_distribution_major_version}}.yml" 15 | - "install.yml" 16 | 17 | - template: src=restuser.service.j2 dest=/etc/systemd/system/restuser.service 18 | 19 | - command: systemctl daemon-reload 20 | 21 | - name: start restuser 22 | service: name=restuser state=restarted enabled=yes 23 | 24 | - name: wait for restuser socket to exist 25 | wait_for: "path=/var/run/restuser.sock" 26 | -------------------------------------------------------------------------------- /users.vault.yml: -------------------------------------------------------------------------------- 1 | $ANSIBLE_VAULT;1.1;AES256 2 | 62343631613339626235333535343964653035316666386538633130623032633264623738333530 3 | 3162343536613039656633663561353235393330623837640a663962616435313236373264376435 4 | 63366131356435323139393132323462643261333362353731396163613434313539353266316236 5 | 3035343135346466320a616330623961326330666537653964336562333364613737336663626638 6 | 36316162343433343432623665663139643065373963643162353864376135633032343236333537 7 | 31336630313466363762303965666431353832616239343861613836373365366165323663306637 8 | 38336562393836386161316561386638376235623030376232363738363761333662306631333465 9 | 35626666613262336666326137363565633161353865333034396233313463613634333738323432 10 | 62653139663235613032653563306633396164316164303437386162393136613539393635343063 11 | 3464623463376633343663393163636131313632323561396162 12 | -------------------------------------------------------------------------------- /roles/common/tasks/ssh.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - fail: msg="no ssh keys defined" 3 | when: github_usernames == [] and other_ssh_keys == [] 4 | 5 | - name: sshd configuration 6 | copy: src=sshd_config dest=/etc/ssh/sshd_config mode=0644 7 | notify: 8 | - restart sshd 9 | 10 | - name: directory to hold public keys 11 | file: state=directory path=/tmp/pubkeys mode=0755 12 | 13 | - name: fetch public keys from github 14 | get_url: dest=/tmp/pubkeys/{{ item }}-pubkeys url=https://github.com/{{ item }}.keys force=yes 15 | with_items: github_usernames 16 | 17 | - name: include additional public keys 18 | template: dest=/tmp/pubkeys/other-pubkeys src=other-pubkeys.j2 19 | 20 | # We assume keys are already set up in the VMs by other scripts 21 | #- name: assemble the authorized keys file 22 | # assemble: dest=/root/.ssh/authorized_keys mode=0600 src=/tmp/pubkeys 23 | -------------------------------------------------------------------------------- /reboot.yml: -------------------------------------------------------------------------------- 1 | - hosts: all 2 | tasks: 3 | - name: test connection (before reboot) 4 | ping: 5 | 6 | - name: reboot! 7 | shell: sleep 2 && shutdown -r now "Ansible updates triggered" 8 | async: 1 9 | poll: 0 10 | become: yes 11 | ignore_errors: true 12 | 13 | - name: wait for SSH port down 14 | local_action: wait_for host={% if 'ansible_ssh_host' in hostvars[inventory_hostname] %}{{ hostvars[inventory_hostname]['ansible_ssh_host'] }}{% else %}{{ inventory_hostname }}{% endif %} port=22 state=stopped 15 | 16 | - name: wait for SSH port up 17 | wait_for: host={% if 'ansible_ssh_host' in hostvars[inventory_hostname] %}{{ hostvars[inventory_hostname]['ansible_ssh_host'] }}{% else %}{{ inventory_hostname }}{% endif %} port=22 state=started delay=30 18 | delegate_to: 127.0.0.1 19 | 20 | - name: test connection (after reboot) 21 | ping: 22 | -------------------------------------------------------------------------------- /deploy.yml: -------------------------------------------------------------------------------- 1 | # This file is not used. It is only here for reference and has 2 | # been split into deploy-part1.yml and deploy-part2.yml. 3 | 4 | --- 5 | - hosts: 6 | - proxy 7 | - jupyterhub_nfs 8 | - jupyterhub_host 9 | - jupyterhub_nodes 10 | vars_files: 11 | - 'secrets.vault.yml' 12 | roles: 13 | - common 14 | 15 | - hosts: 16 | - proxy 17 | - jupyterhub_host 18 | - jupyterhub_nodes 19 | vars_files: 20 | - 'secrets.vault.yml' 21 | roles: 22 | - docker 23 | 24 | - hosts: proxy 25 | vars_files: 26 | - 'secrets.vault.yml' 27 | roles: 28 | - proxy 29 | 30 | - hosts: jupyterhub_nfs 31 | vars_files: 32 | - 'secrets.vault.yml' 33 | roles: 34 | - nfs_server 35 | 36 | - hosts: nfs_clients 37 | vars_files: 38 | - 'secrets.vault.yml' 39 | roles: 40 | - nfs_client 41 | 42 | - hosts: jupyterhub_nodes 43 | vars_files: 44 | - 'secrets.vault.yml' 45 | roles: 46 | - node 47 | 48 | - hosts: jupyterhub_host 49 | vars_files: 50 | - 'secrets.vault.yml' 51 | - 'users.vault.yml' 52 | roles: 53 | - restuser 54 | - hub 55 | -------------------------------------------------------------------------------- /roles/hub/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - fail: msg="jupyterhub_admin_user is not defined" 3 | when: jupyterhub_admin_user == '' 4 | 5 | - include: jupyterhub.yml 6 | - include: swarm.yml 7 | 8 | - name: install docker-compose.yml 9 | template: src=docker-compose.yml dest=/root/docker-compose.yml 10 | 11 | - name: stop docker containers 12 | command: docker-compose stop 13 | 14 | - name: remove docker containers 15 | command: docker-compose rm -f 16 | 17 | - name: pull docker images 18 | command: docker-compose pull 19 | 20 | - name: build docker images 21 | command: docker-compose build 22 | 23 | - name: run jupyterhub 24 | command: docker-compose up -d jupyterhub 25 | 26 | - name: wait for jupyterhub to start 27 | command: docker logs root_jupyterhub_1 28 | register: jpy_status 29 | until: jpy_status.stderr.find("JupyterHub is now running") != -1 30 | retries: 5 31 | delay: 5 32 | 33 | - name: get a jupyterhub api token 34 | command: docker exec root_jupyterhub_1 jupyterhub token -f /srv/jupyterhub_config/jupyterhub_config.py {{ jupyterhub_admin_user }} 35 | register: jpy_api_token 36 | -------------------------------------------------------------------------------- /roles/hub/tasks/jupyterhub.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - fail: msg="configproxy_auth_token is not defined" 3 | when: configproxy_auth_token == '' 4 | - fail: msg="cookie_secret is not defined" 5 | when: cookie_secret == '' 6 | - fail: msg="oauth_client_id is not defined" 7 | when: oauth_client_id == '' 8 | - fail: msg="oauth_client_secret is not defined" 9 | when: oauth_client_secret == '' 10 | - fail: msg="oauth_callback_url is not defined" 11 | when: oauth_callback_url == '' 12 | - fail: msg="oauth_hosted_domain is not defined" 13 | when: oauth_hosted_domain == '' 14 | - fail: msg="admin list is empty" 15 | when: jupyterhub_admins|length == 0 16 | 17 | # Commenting this out because the jupyterhub repository will now be 18 | # preloaded into the hub VM from a local cache. 19 | #- name: clone the dockerfile git repository 20 | # git: repo=https://github.com/triggers/jupyterhub.git dest=/srv/jupyterhub 21 | 22 | - name: create the jupyterhub_users directory 23 | file: path=/srv/jupyterhub_users state=directory 24 | 25 | - name: create the userlist 26 | template: src=userlist.j2 dest=/srv/jupyterhub_users/userlist 27 | -------------------------------------------------------------------------------- /roles/nfs_server_dedicated/tasks/server.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: install nfs packages 3 | yum: name=nfs-utils state=latest 4 | - name: start nfs 5 | service: name=nfs-server state=started enabled=yes 6 | - name: start idmapd 7 | service: name=nfs-idmapd state=started enabled=yes 8 | 9 | - name: create root directory for NFS volume 10 | file: path={{ nfspath }} state=directory 11 | 12 | - name: create sub directory for exchange volume 13 | file: path={{ nfspath }}/exchange state=directory mode=0777 14 | 15 | - name: create sub directory for jupyter volume 16 | file: path={{ nfspath }}/jupyter state=directory mode=0777 17 | 18 | - name: modify /etc/hosts.allow 19 | lineinfile: 20 | dest: /etc/hosts.allow 21 | line: "PORTMAP: {{ hostvars[item]['servicenet_ip'] }}" 22 | with_items: "{{ groups[nfsnodes_group] }}" 23 | 24 | - name: create /etc/exports.d 25 | file: path=/etc/exports.d state=directory 26 | 27 | - name: install /etc/exports.d 28 | template: src=exports dest=/etc/exports.d/{{ nfspath | basename }}.exports mode=0644 29 | 30 | - name: reexport all NFS volume directories 31 | shell: exportfs -ra 32 | 33 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | 5 | # C extensions 6 | *.so 7 | 8 | # Distribution / packaging 9 | .Python 10 | env/ 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | lib/ 17 | lib64/ 18 | parts/ 19 | sdist/ 20 | var/ 21 | *.egg-info/ 22 | .installed.cfg 23 | *.egg 24 | 25 | # PyInstaller 26 | # Usually these files are written by a python script from a template 27 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 28 | *.manifest 29 | *.spec 30 | 31 | # Installer logs 32 | pip-log.txt 33 | pip-delete-this-directory.txt 34 | 35 | # Unit test / coverage reports 36 | htmlcov/ 37 | .tox/ 38 | .coverage 39 | .cache 40 | nosetests.xml 41 | coverage.xml 42 | 43 | # Translations 44 | *.mo 45 | *.pot 46 | 47 | # Django stuff: 48 | *.log 49 | 50 | # Sphinx documentation 51 | docs/_build/ 52 | 53 | # PyBuilder 54 | target/ 55 | 56 | .ipynb_checkpoints 57 | 58 | # Ansible secrets and local overrides 59 | vars.yml 60 | secrets.yml 61 | certificates 62 | /users.yml 63 | /survey.csv 64 | vault-password 65 | old 66 | vars.local.yml 67 | certificates.bak 68 | -------------------------------------------------------------------------------- /roles/nfs_server/tasks/server.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: install nfs packages 3 | yum: name=nfs-utils state=latest 4 | - name: start nfs 5 | service: name=nfs-server state=started enabled=yes 6 | - name: start idmapd 7 | service: name=nfs-idmapd state=started enabled=yes 8 | 9 | - name: create root directory for NFS volume 10 | file: path={{ nfspath }} state=directory 11 | 12 | - name: create sub directory for exchange volume 13 | file: path={{ nfspath }}/exchange state=directory mode=0777 14 | 15 | - name: create sub directory for jupyter volume 16 | file: path={{ nfspath }}/jupyter state=directory mode=0777 17 | 18 | - name: modify /etc/hosts.allow 19 | lineinfile: 20 | dest: /etc/hosts.allow 21 | line: "PORTMAP: {{ hostvars[item]['servicenet_ip'] }}" 22 | with_items: "{{ groups[nfsnodes_group] }}" 23 | when: ansible_os_family == "RedHat" and ansible_distribution_major_version == "7" 24 | 25 | - name: create /etc/exports.d 26 | file: path=/etc/exports.d state=directory 27 | 28 | - name: install /etc/exports.d 29 | template: src=exports dest=/etc/exports.d/{{ nfspath | basename }}.exports mode=0644 30 | 31 | - name: reexport all NFS volume directories 32 | shell: exportfs -ra 33 | 34 | -------------------------------------------------------------------------------- /secrets.vault.yml.example: -------------------------------------------------------------------------------- 1 | --- 2 | # Rename this file to secrets.vault.yml and edit it to include the specific 3 | # keys, certificates, etc. for your deployment. 4 | # 5 | # Note that after you do this, you should encrypt the file with ansible vault 6 | # so that the usernames are not public: 7 | # 8 | # ansible-vault encrypt secrets.vault.yml 9 | 10 | # Overwritten common variables (see common role for details) 11 | github_usernames: [] 12 | other_ssh_keys: [] 13 | 14 | # Overwritten rackspace variables (see rackspace role for details) 15 | rackspace_api_key: '' 16 | rackspace_username: '' 17 | 18 | # Overwritten NFS variables (see the nfs_server and nfs_client roles for details) 19 | nfsserver: '' 20 | nfsdomain: '' 21 | nfspath: '' 22 | exchange: '' 23 | 24 | # Overwritten hub variables (see hub role for details) 25 | configproxy_auth_token: '' 26 | cookie_secret: '' 27 | oauth_client_id: '' 28 | oauth_client_secret: '' 29 | oauth_callback_url: '' 30 | oauth_hosted_domain: '' 31 | jpy_postgres_password: '' 32 | jupyterhub_admin_user: '' 33 | hub_base_url: '' 34 | nbgrader_user: '' 35 | nbgrader_root: '' 36 | assignments_repo_name: '' 37 | assignments_repo_url: '' 38 | assignments_repo_clone_url: '' 39 | gradebook_postgres_password: '' 40 | -------------------------------------------------------------------------------- /roles/docker/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # This is where the TLS certificates and keys should be stored for Docker 3 | docker_tls_path: /root/.docker 4 | 5 | # Whether or not the Docker server should run on a public port with TLS 6 | docker_server_tls: no 7 | # If running on a public port with TLS, this is the port number 8 | docker_server_port: 2376 9 | 10 | # Whether or not the Docker client needs TLS (to communicate with a Docker 11 | # server running TLS) 12 | docker_client_tls: no 13 | 14 | # This is the certificate authority for the Docker certs. This should never 15 | # actually be saved in plain text, it should be overwritten in an encrypted 16 | # vault file. 17 | docker_ca_cert: '' 18 | 19 | # This should contain TLS certificates for each of the machines that use 20 | # Docker. Each key should correspond to the name of the host in the inventory 21 | # file. These should never actually be saved in plain text, they should be 22 | # overwritten in an encrypted vault file. 23 | docker_tls_cert: '' 24 | 25 | # This should contain TLS keys for each of the machines that use Docker, and 26 | # there should be one for each certificate in `tls_certs`. These should never 27 | # actually be saved in plain text, they should be overwritten in an encrypted 28 | # vault file. 29 | docker_tls_key: '' 30 | -------------------------------------------------------------------------------- /roles/proxy/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: clone the dockerfile git repository 3 | git: repo=https://github.com/triggers/systemuser.git dest=/srv/systemuser 4 | 5 | - name: configuration directories 6 | file: state=directory dest=/srv/nginx mode=0755 7 | 8 | - name: SSL credentials 9 | template: src={{ item.from }} dest={{ item.to }} mode=0644 10 | with_items: 11 | - from: ssl.key.j2 12 | to: "{{ ssl_key_path }}" 13 | - from: ssl.crt.j2 14 | to: "{{ ssl_cert_path }}" 15 | 16 | - name: nginx configuration 17 | template: src=nginx.conf.j2 dest=/srv/nginx/nginx.conf mode=0644 18 | 19 | - name: copy the Dockerfile to the nginx configuration directory 20 | template: src=Dockerfile.j2 dest=/srv/nginx/Dockerfile mode=0644 21 | 22 | - name: install docker-compose.yml 23 | template: src=docker-compose.yml dest=/root/docker-compose.yml 24 | 25 | - name: stop docker containers 26 | command: docker-compose stop 27 | 28 | - name: remove docker containers 29 | command: docker-compose rm -f 30 | 31 | ## This image is now preloaded before ansible runs, but 32 | ## ansible still tries to contact docker.io to find a later 33 | ## image. This pull only affects the staticfiles container. 34 | #- name: pull docker images 35 | # command: docker-compose pull 36 | 37 | - name: build docker images 38 | command: docker-compose build 39 | 40 | - name: start services 41 | command: docker-compose up -d 42 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2017-2018, National Institute of Informatics. 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | * Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | * Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | * Neither the name of the copyright holder nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /roles/hub/templates/docker-compose.yml: -------------------------------------------------------------------------------- 1 | swarm: 2 | image: dockerswarm/swarm 3 | restart: always 4 | command: manage --strategy={{ docker_swarm_strategy }} --tlsverify --tlscacert={{ docker_tls_path }}/ca.pem --tlscert={{ docker_tls_path }}/cert.pem --tlskey={{ docker_tls_path }}/key.pem file:///root/.swarm/cluster 5 | volumes: 6 | - /root/.swarm:/root/.swarm 7 | - "{{ docker_tls_path }}:{{ docker_tls_path }}" 8 | ports: 9 | - "127.0.0.1:2375:2375" 10 | 11 | jpydb: 12 | image: postgres 13 | restart: always 14 | environment: 15 | POSTGRES_DB: jupyterhub 16 | POSTGRES_PASSWORD: "{{ jpy_postgres_password }}" 17 | PGDATA: /var/lib/postgresql/data/jupyterhub 18 | volumes: 19 | - /srv/postgres/jupyterhub:/var/lib/postgresql/data/jupyterhub 20 | 21 | jupyterhub: 22 | build: /srv/jh-image-wrap 23 | restart: always 24 | environment: 25 | OAUTH_CLIENT_ID: "{{ oauth_client_id }}" 26 | OAUTH_CLIENT_SECRET: "{{ oauth_client_secret }}" 27 | OAUTH_CALLBACK_URL: "{{ oauth_callback_url }}" 28 | HOSTED_DOMAIN: "{{ oauth_hosted_domain }}" 29 | JPY_COOKIE_SECRET: "{{ cookie_secret }}" 30 | JPY_DB_USER: postgres 31 | JPY_DB_PASSWORD: "{{ jpy_postgres_password }}" 32 | JPY_DB_NAME: jupyterhub 33 | CONFIGPROXY_AUTH_TOKEN: "{{ configproxy_auth_token }}" 34 | DOCKER_TLS_CERT: "{{ docker_tls_path }}/cert.pem" 35 | DOCKER_TLS_KEY: "{{ docker_tls_path }}/key.pem" 36 | HUB_IP: "{{ servicenet_ip }}" 37 | NBGRADER_EXCHANGE: "{{ exchange }}" 38 | volumes: 39 | - /srv/jupyterhub_users:/srv/jupyterhub_users 40 | - "{{ docker_tls_path }}:{{ docker_tls_path }}" 41 | - /var/run/restuser.sock:/restuser.sock 42 | ports: 43 | - "{{ servicenet_ip }}:8000:8000" 44 | - "{{ servicenet_ip }}:8081:8081" 45 | links: 46 | - swarm 47 | - jpydb:postgres 48 | -------------------------------------------------------------------------------- /roles/proxy/templates/nginx.conf.j2: -------------------------------------------------------------------------------- 1 | # {{ ansible_managed }} 2 | 3 | worker_processes {{ ansible_processor_count*2 }}; 4 | 5 | events { 6 | worker_connections 1024; 7 | } 8 | 9 | http { 10 | 11 | include /etc/nginx/mime.types; 12 | default_type application/octet-stream; 13 | 14 | server { 15 | listen 80; 16 | server_name {{ inventory_hostname }}; 17 | rewrite ^ https://$host$request_uri? permanent; 18 | } 19 | 20 | server { 21 | listen 443; 22 | 23 | client_max_body_size 50M; 24 | 25 | server_name {{ inventory_hostname }}; 26 | 27 | ssl on; 28 | ssl_certificate {{ ssl_cert_path }}; 29 | ssl_certificate_key {{ ssl_key_path }}; 30 | 31 | ssl_ciphers "AES128+EECDH:AES128+EDH"; 32 | ssl_protocols TLSv1 TLSv1.1 TLSv1.2; 33 | ssl_prefer_server_ciphers on; 34 | ssl_session_cache shared:SSL:10m; 35 | add_header Strict-Transport-Security "max-age=63072000; includeSubDomains"; 36 | add_header X-Content-Type-Options nosniff; 37 | ssl_stapling on; # Requires nginx >= 1.3.7 38 | ssl_stapling_verify on; # Requires nginx => 1.3.7 39 | resolver_timeout 5s; 40 | 41 | # Expose logs to "docker logs". 42 | # See https://github.com/nginxinc/docker-nginx/blob/master/Dockerfile#L12-L14 43 | access_log /var/log/nginx/access.log; 44 | error_log /var/log/nginx/error.log; 45 | 46 | location ~ /(user-[a-zA-Z0-9]*)/static(.*) { 47 | alias {{ notebook_static_files }}$2; 48 | } 49 | 50 | location / { 51 | proxy_pass http://{{ jupyterhub_host }}:8000; 52 | 53 | proxy_set_header X-Real-IP $remote_addr; 54 | proxy_set_header Host $host; 55 | proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; 56 | 57 | proxy_set_header X-NginX-Proxy true; 58 | } 59 | 60 | location ~* /(user/[^/]*)/(api/kernels/[^/]+/channels|terminals/websocket)/? { 61 | proxy_pass http://{{ jupyterhub_host }}:8000; 62 | 63 | proxy_set_header X-Real-IP $remote_addr; 64 | proxy_set_header Host $host; 65 | proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; 66 | 67 | proxy_set_header X-NginX-Proxy true; 68 | 69 | # WebSocket support 70 | proxy_http_version 1.1; 71 | proxy_set_header Upgrade $http_upgrade; 72 | proxy_set_header Connection "upgrade"; 73 | proxy_read_timeout 86400; 74 | 75 | } 76 | } 77 | 78 | } 79 | -------------------------------------------------------------------------------- /roles/common/files/sshd_config: -------------------------------------------------------------------------------- 1 | # Package generated configuration file 2 | # See the sshd_config(5) manpage for details 3 | 4 | # What ports, IPs and protocols we listen for 5 | Port 22 6 | # Use these options to restrict which interfaces/protocols sshd will bind to 7 | #ListenAddress :: 8 | #ListenAddress 0.0.0.0 9 | Protocol 2 10 | # HostKeys for protocol version 2 11 | HostKey /etc/ssh/ssh_host_rsa_key 12 | HostKey /etc/ssh/ssh_host_dsa_key 13 | HostKey /etc/ssh/ssh_host_ecdsa_key 14 | #Privilege Separation is turned on for security 15 | UsePrivilegeSeparation yes 16 | 17 | # Lifetime and size of ephemeral version 1 server key 18 | KeyRegenerationInterval 3600 19 | ServerKeyBits 768 20 | 21 | # Logging 22 | SyslogFacility AUTH 23 | LogLevel INFO 24 | 25 | # Authentication: 26 | LoginGraceTime 120 27 | PermitRootLogin yes 28 | StrictModes yes 29 | 30 | RSAAuthentication yes 31 | PubkeyAuthentication yes 32 | #AuthorizedKeysFile %h/.ssh/authorized_keys 33 | 34 | # Don't read the user's ~/.rhosts and ~/.shosts files 35 | IgnoreRhosts yes 36 | # For this to work you will also need host keys in /etc/ssh_known_hosts 37 | RhostsRSAAuthentication no 38 | # similar for protocol version 2 39 | HostbasedAuthentication no 40 | # Uncomment if you don't trust ~/.ssh/known_hosts for RhostsRSAAuthentication 41 | #IgnoreUserKnownHosts yes 42 | 43 | # To enable empty passwords, change to yes (NOT RECOMMENDED) 44 | PermitEmptyPasswords no 45 | 46 | # Change to yes to enable challenge-response passwords (beware issues with 47 | # some PAM modules and threads) 48 | ChallengeResponseAuthentication no 49 | 50 | # Change to no to disable tunnelled clear text passwords 51 | PasswordAuthentication no 52 | 53 | # Kerberos options 54 | #KerberosAuthentication no 55 | #KerberosGetAFSToken no 56 | #KerberosOrLocalPasswd yes 57 | #KerberosTicketCleanup yes 58 | 59 | # GSSAPI options 60 | #GSSAPIAuthentication no 61 | #GSSAPICleanupCredentials yes 62 | 63 | X11Forwarding no 64 | # X11DisplayOffset 10 65 | PrintMotd no 66 | PrintLastLog yes 67 | TCPKeepAlive yes 68 | #UseLogin no 69 | 70 | #MaxStartups 10:30:60 71 | #Banner /etc/issue.net 72 | 73 | # Allow client to pass locale environment variables 74 | # AcceptEnv LANG LC_* 75 | 76 | Subsystem sftp /usr/lib/openssh/sftp-server 77 | 78 | # Set this to 'yes' to enable PAM authentication, account processing, 79 | # and session processing. If this is enabled, PAM authentication will 80 | # be allowed through the ChallengeResponseAuthentication and 81 | # PasswordAuthentication. Depending on your PAM configuration, 82 | # PAM authentication via ChallengeResponseAuthentication may bypass 83 | # the setting of "PermitRootLogin without-password". 84 | # If you just want the PAM account and session checks to run without 85 | # PAM authentication, then enable this but set PasswordAuthentication 86 | # and ChallengeResponseAuthentication to 'no'. 87 | UsePAM yes 88 | -------------------------------------------------------------------------------- /roles/hub/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Needed to make sure the API version matches with docker-py 3 | docker_api_version: 1.16 4 | 5 | # This is the auth token for JupyterHub. You can create it with: 6 | # openssl rand -hex 16 7 | # Note that this should never be stored in plaintext; you should probably 8 | # override it in an encrypted vault file. 9 | configproxy_auth_token: '' 10 | 11 | # This is the cookie secret for JupyterHub. You can create it with: 12 | # openssl rand -hex 2048 13 | # Note that this should never be stored in plaintext; you should probably 14 | # override it in an encrypted vault file. 15 | cookie_secret: '' 16 | 17 | # Get these from the Google OAuth application. Note that these should never 18 | # be stored in plaintext, they should be overwritten in an encrypted vault file. 19 | oauth_client_id: '' 20 | oauth_client_secret: '' 21 | oauth_callback_url: '' 22 | oauth_hosted_domain: '' 23 | 24 | # This is the password for the 'postgres' user for the JupyterHub database (the 25 | # database name will be 'jupyterhub') 26 | jpy_postgres_password: '' 27 | 28 | # This is the name of an admin user of the JupyterHub instance, necessary 29 | # to get an API key from JupyterHub 30 | jupyterhub_admin_user: '' 31 | 32 | # Root url of the hub (e.g. 'https://compmodels.tmpnb.org/') 33 | hub_base_url: '' 34 | 35 | # The name of the repository containing the homework assignments, the public URL 36 | # for the repository, and a private URL from which to clone the repository. 37 | # These two URLs can be the same, but if you have a private repo then the clone 38 | # URL can take the form of 39 | # 'https://:@github.com/org/repo.git' rather than 40 | # 'https://github.com/org/repo.git'. The clone URL will be used to clone the 41 | # repo, but then the remote will be set to the public URL, so that your access 42 | # token isn't stored anywhere. 43 | assignments_repo_name: '' 44 | assignments_repo_url: '' 45 | assignments_repo_clone_url: '' 46 | 47 | # This is the path to where the docker TLS certificates are stored -- this is 48 | # necessary for the SwarmSpawner to actually be able to communicate with swarm 49 | docker_tls_path: /root/.docker 50 | 51 | # These are the usernames that will have JupyterHub admin access. This list 52 | # should probably be overwritten in an encrypted vault file. 53 | jupyterhub_admins: [] 54 | 55 | # These are the usernames that have access to JupyterHub, but not admin access. 56 | # This list should probably be overwritten in an encrypted vault file. 57 | jupyterhub_users: [] 58 | 59 | # This is where the TLS certificates and keys should be stored for Docker 60 | docker_tls_path: /root/.docker 61 | 62 | # This is the port that swarm will attempt to connect to for each of the 63 | # Docker hosts 64 | docker_server_port: 2376 65 | 66 | # This is the strategy that swarm will use to allocate containers. For details 67 | # on the different options, see the swarm documentation. 68 | docker_swarm_strategy: spread 69 | -------------------------------------------------------------------------------- /INSTALL.md: -------------------------------------------------------------------------------- 1 | # Installation and setup instructions 2 | 3 | :warning: Note: you'll need to install Ansible from source (`devel` branch) to 4 | get the current versions of the Docker modules. 5 | 6 | ## Generating TLS/SSL certificates 7 | 8 | **This only ever needs to be done once!** 9 | 10 | You'll need to generate SSL/TLS certificates for the hub server and node servers. 11 | To do this, you can use the [keymaster](https://github.com/cloudpipe/keymaster) docker container. 12 | First, setup the certificates directory, password, and certificate authority: 13 | 14 | ``` 15 | mkdir certificates 16 | 17 | touch certificates/password 18 | chmod 600 certificates/password 19 | cat /dev/random | head -c 128 | base64 > certificates/password 20 | 21 | KEYMASTER="docker run --rm -v $(pwd)/certificates/:/certificates/ cloudpipe/keymaster" 22 | 23 | ${KEYMASTER} ca 24 | ``` 25 | 26 | Then, to generate a keypair for a server: 27 | 28 | ``` 29 | ${KEYMASTER} signed-keypair -n server1 -h server1.website.com -p both -s IP:192.168.0.1 30 | ``` 31 | 32 | where `server1` is the name of a server (e.g. `compmodels`), `server1.website.com` is the hostname for the server, and `192.168.0.1` is the IP address of the server. 33 | 34 | You'll need to generate keypairs for the hub server and for each of the node servers. 35 | 36 | ## Secrets 37 | 38 | Copy the `secrets.vault.yml.example` file to `secrets.vault.yml` and edit it to include keys and passwords for your specific deployment. You should then encrypt it using Ansible vault: 39 | 40 | ``` 41 | ansible-vault encrypt secrets.vault.yml 42 | ``` 43 | 44 | It may be helpful to put your Ansible vault password in a file called `vault-password`, so then you can do: 45 | 46 | ``` 47 | ansible-vault encrypt --vault-password-file vault-password secrets.vault.yml 48 | ``` 49 | 50 | You will also need to save host-specific variables (such as certificates) into the `host_vars` directory (one file per host in the inventory). There is a file there called `example` that will tell you what variables need to be defined in those files. One example of something that needs to be in these files are the certificates for the servers running Docker with TLS. If you generated certificates as suggested above, then you will have all your Docker certs in the `certificates` folder. You can copy these to the `host_vars` directory with the helper script `script/assemble_certs` -- you will just need to edit the `assemble_certs` script so that it uses the correct names for your certificates and hosts. 51 | 52 | ## Users 53 | 54 | For the whitelist of users, you need to copy `users.vault.yml.example` to `users.vault.yml` and edit it to include your list of admins and users. Once you are done editing it, you should encrypt it using Ansible vault: 55 | 56 | ``` 57 | ansible-vault encrypt --vault-password-file vault-password users.vault.yml 58 | ``` 59 | 60 | ## Other variables 61 | 62 | You'll need to set a few other variables. Copy `vars.yml.example` to `vars.yml` and edit it to include specifics for your deployment. 63 | -------------------------------------------------------------------------------- /script/assemble_certs: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | """Reads in the TLS certificates and keys for all the Docker hosts, and saves 4 | them to the secrets.vault.yml file. 5 | 6 | """ 7 | 8 | import yaml 9 | import subprocess as sp 10 | import os 11 | import contextlib 12 | from collections import OrderedDict 13 | 14 | ################################################################################ 15 | 16 | # This is the root directory of the ansible config 17 | root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) 18 | 19 | # This is the filename where the vault password is stored 20 | vault_password_file = os.path.join(root, 'vault-password') 21 | 22 | # This is the directory containing the certificates 23 | certificates_dir = os.path.join(root, 'certificates') 24 | 25 | # This is the name of the certificate authority (not including the extension) 26 | ca_name = "ca" 27 | 28 | # If your certificates have different names than the hosts in your inventory 29 | # file, then you can use this name map to rename them. The keys are the names 30 | # in the hosts file, the keys are the base name of the certificate (e.g., if 31 | # the certificate and key are `hub-cert.pem` and `hub-key.pem`, respectively, 32 | # then the base name is 'hub'. 33 | name_map = { 34 | "hub": "hub", 35 | "node1": "node1", 36 | "node2": "node2" 37 | } 38 | 39 | ################################################################################ 40 | 41 | # from http://stackoverflow.com/questions/8640959/how-can-i-control-what-scalar-form-pyyaml-uses-for-my-data 42 | class literal(str): pass 43 | 44 | def literal_presenter(dumper, data): 45 | return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='|') 46 | yaml.add_representer(literal, literal_presenter) 47 | 48 | def ordered_dict_presenter(dumper, data): 49 | return dumper.represent_dict(data.items()) 50 | yaml.add_representer(OrderedDict, ordered_dict_presenter) 51 | 52 | 53 | @contextlib.contextmanager 54 | def vault(filename): 55 | if os.path.exists(filename): 56 | sp.call([ 57 | "ansible-vault", "decrypt", filename, 58 | "--vault-password-file", vault_password_file 59 | ]) 60 | 61 | yield 62 | 63 | sp.call([ 64 | "ansible-vault", "encrypt", filename, 65 | "--vault-password-file", vault_password_file 66 | ]) 67 | 68 | 69 | for host in name_map: 70 | filename = os.path.join("host_vars", host) 71 | with vault(filename): 72 | # read the secrets in 73 | if os.path.exists(filename): 74 | with open(filename, 'r') as fh: 75 | secrets = yaml.load(fh.read()) 76 | else: 77 | secrets = {} 78 | 79 | # save the certificate authority 80 | with open(os.path.join(certificates_dir, '{}.pem'.format(ca_name)), "r") as fh: 81 | secrets["docker_ca_cert"] = literal(fh.read()) 82 | 83 | # save the TLS certificate 84 | with open(os.path.join(certificates_dir, '{}-cert.pem'.format(name_map[host])), 'r') as fh: 85 | secrets["docker_tls_cert"] = literal(fh.read()) 86 | 87 | # save the TLS key 88 | with open(os.path.join(certificates_dir, '{}-key.pem'.format(name_map[host])), 'r') as fh: 89 | secrets["docker_tls_key"] = literal(fh.read()) 90 | 91 | # save the secrets back out 92 | with open(filename, "w") as fh: 93 | fh.write("---\n") 94 | fh.write(yaml.dump(secrets, default_flow_style=False)) 95 | -------------------------------------------------------------------------------- /roles/docker/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - fail: msg="docker_ca_cert is undefined" 3 | when: (docker_client_tls or docker_server_tls) and docker_ca_cert == '' 4 | - fail: msg="docker_tls_cert is undefined" 5 | when: (docker_client_tls or docker_server_tls) and docker_tls_cert == '' 6 | - fail: msg="docker_tls_key is undefined" 7 | when: (docker_client_tls or docker_server_tls) and docker_tls_key == '' 8 | 9 | - name: tweak grub settings 10 | lineinfile: 11 | dest: /etc/default/grub 12 | regexp: GRUB_CMDLINE_LINUX=.* 13 | line: GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1" 14 | register: grubbed 15 | 16 | - name: apply changed grub settings 17 | command: update-grub 18 | when: grubbed | changed 19 | 20 | - name: reboot 21 | command: shutdown -r now "Ansible updated grub" 22 | async: 0 23 | poll: 0 24 | ignore_errors: true 25 | when: grubbed | changed 26 | 27 | - name: wait for server to relaunch 28 | local_action: wait_for host={{ ansible_ssh_host }} state=started 29 | when: grubbed | changed 30 | 31 | # The next task needs to be run after reboot so that uname picks the 32 | # version of the kernel that is going to be used. The purpose is to 33 | # get aufs installed, so that docker will use aufs. Otherwise, it 34 | # will default to devmapper, which has race conditions. 35 | # https://github.com/docker/docker/issues/4036 36 | 37 | - name: install correct linux-image-extra 38 | shell: apt-get -y install linux-image-extra-$(uname -r) 39 | 40 | - name: see if HTTPS transport is already present 41 | stat: path=/usr/lib/apt/methods/https get_md5=false 42 | register: https_transport_file 43 | 44 | - name: ensure HTTPS transport is available to apt 45 | apt: update_cache=yes cache_valid_time=600 name=apt-transport-https 46 | when: not https_transport_file.stat.exists 47 | 48 | - name: ensure the docker apt key is trusted 49 | apt_key: 50 | keyserver: hkp://keyserver.ubuntu.com:80 51 | id: 36A1D7869245C8950F966E92D8576A8BA88D21E9 52 | state: present 53 | 54 | - name: ensure the docker apt repository is present 55 | apt_repository: 56 | repo: deb https://get.docker.com/ubuntu docker main 57 | state: present 58 | 59 | - name: install docker 60 | apt: update_cache=yes name=lxc-docker 61 | 62 | - name: install docker configuration 63 | template: src=docker_config dest=/etc/default/docker mode=0644 64 | 65 | - name: create docker TLS directory 66 | file: path={{ docker_tls_path }} state=directory 67 | when: docker_client_tls or docker_server_tls 68 | 69 | - name: install docker TLS credentials 70 | template: src={{ item.from }} dest={{ item.to }} mode=0644 71 | with_items: 72 | - from: ca.pem.j2 73 | to: "{{ docker_tls_path }}/ca.pem" 74 | - from: key.pem.j2 75 | to: "{{ docker_tls_path }}/key.pem" 76 | - from: cert.pem.j2 77 | to: "{{ docker_tls_path }}/cert.pem" 78 | when: docker_client_tls or docker_server_tls 79 | 80 | - name: restart docker 81 | shell: service docker restart 82 | 83 | - name: docker-py 84 | pip: name=docker-py state=present version=1.1.0 85 | 86 | - name: stop running docker containers 87 | shell: docker stop $(docker ps -a -q) || true 88 | 89 | - name: remove docker containers 90 | shell: docker rm -v $(docker ps -a -q) || true 91 | 92 | - name: run docker cleanup cronjob 93 | cron: name="docker cleanup" minute=15 user="root" job="/usr/bin/docker rmi $(/usr/bin/docker images -q --filter 'dangling=true')" 94 | 95 | - name: install docker-compose 96 | shell: curl -L https://github.com/docker/compose/releases/download/1.3.3/docker-compose-`uname -s`-`uname -m` > /usr/local/bin/docker-compose && chmod +x /usr/local/bin/docker-compose 97 | 98 | - name: allow incoming ufw connections 99 | lineinfile: 100 | dest: /etc/default/ufw 101 | regexp: DEFAULT_FORWARD_POLICY=.* 102 | line: DEFAULT_FORWARD_POLICY="ACCEPT" 103 | register: ufw_changed 104 | 105 | - name: reload ufw 106 | shell: ufw reload 107 | when: ufw_changed | changed 108 | -------------------------------------------------------------------------------- /script/launch.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | #!/usr/bin/env python 5 | # -*- coding: utf-8 -*- 6 | ''' 7 | This script requires 4 environment variables to be declared: 8 | 9 | OS_USERNAME - Rackspace user for account that servers will be launched on 10 | OS_PASSWORD - API Key for the server launch user 11 | 12 | OS_DNS_USERNAME - Rackspace user with the tmpnb.org domain 13 | OS_DNS_PASSWORD - API key for the DNS user 14 | 15 | Then to run, you specify which node number we're creating like compmodels-iad-001.tmpnb.org 16 | 17 | python script/launch.py 10 18 | 19 | The Ansible inventory file is spat out to stdout at the end. 20 | ''' 21 | 22 | import binascii 23 | import os 24 | 25 | import pyrax 26 | 27 | def name_new_nodes(prefix="compmodels", region="iad", node_num=1, domain="tmpnb.org"): 28 | # The naming problem 29 | node_naming_scheme = "{prefix}-{region}-{node_num:03}" 30 | node_base_name = node_naming_scheme.format(**locals()) 31 | 32 | user_server_name = node_base_name + "-user" + "." + domain 33 | proxy_server_name = node_base_name + "." + domain 34 | 35 | return user_server_name, proxy_server_name 36 | 37 | def launch_node(prefix="compmodels", region="iad", node_num=1, domain="tmpnb.org"): 38 | key_name = "team" 39 | 40 | pyrax.set_setting("identity_type", "rackspace") 41 | pyrax.set_credentials(os.environ["OS_USERNAME"], os.environ["OS_PASSWORD"]) 42 | 43 | cs = pyrax.connect_to_cloudservers(region=region.upper()) 44 | 45 | # My least favorite bug in pyrax - silent errors 46 | if(cs is None): 47 | raise Exception("Unable to connect to given region '{}'".format(region)) 48 | 49 | # Get our base images 50 | images = cs.list_base_images() 51 | ubs = [image for image in images if "Ubuntu 14.04" in image.name] 52 | user_image = [image for image in ubs if "OnMetal" in image.name][0] 53 | proxy_image = [image for image in ubs if "PVHVM" in image.name][0] 54 | 55 | user_server_name, proxy_server_name = name_new_nodes(prefix=prefix, 56 | region=region.lower(), 57 | node_num=node_num, 58 | domain=domain) 59 | 60 | # Launch the servers 61 | proxy_server = cs.servers.create(proxy_server_name, image=proxy_image.id, flavor='performance2-15', key_name=key_name) 62 | user_server = cs.servers.create(user_server_name, image=user_image.id, flavor='onmetal-compute1', key_name=key_name) 63 | 64 | # Wait on them 65 | print("Waiting on Proxy server") 66 | proxy_server = pyrax.utils.wait_for_build(proxy_server, verbose=True) 67 | print("Waiting on jupyterhub user server") 68 | user_server = pyrax.utils.wait_for_build(user_server, verbose=True) 69 | 70 | # Making this in case we want some JSON 71 | node_layout = { 72 | 'jupyterhub_server': { 73 | 'private': user_server.networks['private'][0], 74 | 'public': user_server.networks['public'][0] 75 | }, 76 | 'proxy_server': { 77 | 'public': proxy_server.networks['public'][0] 78 | } 79 | } 80 | 81 | inventory = '''[jupyterhub] 82 | {user_server_name} ansible_ssh_user=root ansible_ssh_host={jupyterhub_server_public} 83 | 84 | [proxy] 85 | {proxy_server_name} ansible_ssh_user=root ansible_ssh_host={proxy_server_public} jupyterhub_host={jupyterhub_server_private} 86 | '''.format(jupyterhub_server_public=user_server.accessIPv4, 87 | jupyterhub_server_private=user_server.networks['private'][0], 88 | proxy_server_public=proxy_server.accessIPv4, 89 | user_server_name=user_server_name, 90 | proxy_server_name=proxy_server_name, 91 | ) 92 | 93 | print(inventory) 94 | 95 | # If a separate account is used for DNS, use that instead 96 | if("OS_DNS_USERNAME" in os.environ and "OS_DNS_PASSWORD" in os.environ): 97 | pyrax.set_credentials(os.environ["OS_DNS_USERNAME"], os.environ["OS_DNS_PASSWORD"]) 98 | 99 | dns = pyrax.cloud_dns.find(name=domain) 100 | dns.add_record({'type': 'A', 101 | 'name': proxy_server_name, 102 | 'ttl': 60*5, 103 | 'data': proxy_server.accessIPv4}) 104 | 105 | 106 | if __name__ == "__main__": 107 | import argparse 108 | 109 | parser = argparse.ArgumentParser(description='Launch nodes for tmpnb') 110 | 111 | parser.add_argument('prefix', type=str, default='compmodels', 112 | help='prefix in the URL base') 113 | parser.add_argument('region', type=str, default='iad', 114 | help='region to deploy to, also part of the domain name') 115 | parser.add_argument('node_num', metavar='N', type=int, 116 | help='what this set of servers will be identified as numerically') 117 | parser.add_argument('domain', type=str, default="tmpnb.org", 118 | help='domain to host the servers on') 119 | 120 | args = parser.parse_args() 121 | launch_node(prefix=args.prefix, 122 | region=args.region, 123 | node_num=args.node_num, 124 | domain=args.domain 125 | ) 126 | 127 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # JupyterHub deployment for the compmodels class 2 | 3 | This repository contains an Ansible playbook for launching JupyterHub for the 4 | Computational Models of Cognition class at Berkeley. 5 | 6 | The setup is a bit complex, so this readme will try to break it down and explain everything that's going on. If you want to jump right into the details of how to deploy, see first the [installation instructions](INSTALL.md) and then the section on [deploying](#deploying). 7 | 8 | ## Overview 9 | 10 | To understand what's going on behind the scenes, it is probably helpful to know what happens when a user accesses the server. 11 | 12 | 1. First, they go to the main url for the server. 13 | 2. This url actually points to a proxy server which authenticates the SSL connection, and proxies the connection to the JupyterHub instance running on the hub server. 14 | 3. The hub server is both a NFS server (to serve user's home directories) and the JupyterHub server. JupyterHub runs in a docker container called "jupyterhub". 15 | 4. Users see a JupyterHub login screen. When they click login, they are authenticated. When they access their server, JupyterHub creates a new docker container on one of the node servers running an IPython notebook server. This docker container is called "jupyter-username", where "username" is the user's username. 16 | 5. As users open IPython notebooks and run them, they are actually communicating with one of the seven node servers. The URL still appears the same, because the connection is first being proxied to the hub server via the proxy server, and then proxied a second time to the node server via the JupyterHub proxy. 17 | 6. Users have access to their home directory, because each node server is also a NFS client with the filesystem mounted at `/home`. 18 | 19 | ## Proxy server 20 | 21 | The proxy server runs [nginx](http://nginx.org/en/) (pronounced "engine x"), which is a reverse HTTP proxy, in a docker container called "nginx". 22 | It proxies all connections from the main URL to port 8000 on the hub server. 23 | 24 | You can verify that the proxy is running with `docker ps`. 25 | If you need to access logs for the proxy server, you can run `docker logs --tail=10 nginx`, and adjust the tail length as needed. 26 | 27 | ### Static files 28 | 29 | nginx proxies to the hub server, but it also serves the static notebook files directly to save on the number of files that need to be accessed. 30 | To do this, it mounts a volume from the `jupyter/systemuser` container, which contains all the necessary notebook files. 31 | However, this also means that if the user server docker images are updated on the node servers, that the `jupyter/systemuser` container must *also* be updated on the proxy server or you may see weird errors because the clients are requesting files that the proxy server has an old version of, or doesn't have at all. 32 | See [the docker documentation on data volume containers](https://docs.docker.com/userguide/dockervolumes/#creating-and-mounting-a-data-volume-container) for more details on how this works. 33 | 34 | ## Hub server 35 | 36 | The hub server fulfills two functions: it is both the NFS server, hosting all user's home directories, and it is the JupyterHub server. 37 | 38 | ### NFS 39 | 40 | [NFS](http://en.wikipedia.org/wiki/Network_File_System) is the Network File System. 41 | It requires there to be a host server, where the files actually exist, and then any number of client servers that mount the NFS filesystem. 42 | Once the filesystem is mounted, it behaves just like a normal filesystem. 43 | 44 | The hub server on its own only has about 28GB of disk space, so to supplement this, we have about 3TB of additional storage mounted at `/var/lib/docker`. 45 | 46 | The original copies of all files are located at `/var/lib/docker/export/home` (the directory `/var/lib/docker/export` is the root of the NFS host filesystem). 47 | They are additionally mounted at `/home` on the hub server, and that is the location from which they should *always* be accessed. 48 | 49 | There is a cron job that runs every hour to back up the files in `/home` using the script `/srv/backup/backup.sh`. 50 | The script runs [duplicity](http://duplicity.nongnu.org/) once every hour, which is a backup service that performs a full backup every seven days and otherwise performs incremental backups. 51 | The files get encrypted and then backed up to a Rackspace Cloud Files container, from which they can also be restored, if necessary. 52 | In addition, another script (`/srv/backup/cleanup.sh`) gets run once a day, and will remove old backups. Only the most recent two full backups are kept. 53 | 54 | Logs for the duplicity service are in `/srv/backup/duplicity.log`. 55 | Logs for NFS don't have their own log file; they can be found just in `/var/log/syslog`. 56 | 57 | ### JupyterHub 58 | 59 | The JupyterHub setup itself is actually pretty straightforward. 60 | JupyterHub runs in a docker container from the image `compmodels/jupyterhub`, which is built from `jupyter/jupyterhub`. 61 | The differences are that our version of JupyterHub uses a special blend of Google authentication with local system users and spawns user servers inside docker containers on the node servers using a spawner based on the [system user docker spawner](https://github.com/jupyter/dockerspawner). 62 | What this basically means is: 63 | 64 | 1. Users are created on the hub server with a username that needs to be the same as their Google username. 65 | 2. In addition to their username, JupyterHub stores the pid of each user. 66 | 3. When they login, JupyterHub authenticates the users with Google. 67 | 4. Once authenticated, JupyterHub spawns a docker container on one of the node servers and mounts the user's home directory inside the container. 68 | 5. A user is created inside the docker container with the appropriate username and pid, so that they have access to the files in their home directory. 69 | 70 | You should be able to see the JupyterHub docker container with `docker ps`. To get the JupyterHub logs, you can run `docker logs --tail=10 jupyterhub`. 71 | 72 | ### Restuser 73 | 74 | When a new user is added to JupyterHub, it ensures that the user exists on the system by communicating with a service called [restuser](https://github.com/minrk/restuser). 75 | This is just a simple REST API that can create new users on the system. 76 | This is necessary, because JupyterHub is running in a docker container, and so can't actually create users on the system directly. 77 | 78 | When users are created, their home directory is created on the NFS mount. 79 | The skeleton directory used to initialize their home directory can be found at `/srv/skeldir`. 80 | Logs for the restuser service can be found at `/var/log/restuser.log`. 81 | 82 | ### Swarm 83 | 84 | To spawn the user docker containers, we use a load-balancing clustering system called [swarm](https://github.com/docker/swarm). 85 | Essentially, swarm exposes an interface that looks just like the normal docker interface, and figures out in the background where to start new containers. 86 | For example, to get a list of the docker containers that are running through swarm (i.e., the user containers that are running on all node servers): 87 | 88 | ``` 89 | docker --tls -H 127.0.0.1:2376 ps 90 | ``` 91 | 92 | To get the logs for a container running through swarm, you can run `docker --tls -H 127.0.0.1:2376 logs --tail=10 jupyter-username`, where `username` is the username for the user that you want logs from. 93 | 94 | Swarm itself also runs in a docker container, so you can get the logs with `docker logs --tail=10 swarm`. 95 | 96 | ### Culling containers 97 | 98 | Each docker container started through swarm is allocated 1GB of memory (currently there is no limit on CPU, but if a problem arises, we can institute one as well). 99 | Each node server has 32GB of memory, meaning that we can run about 31*7=217 containers at once. 100 | We actually have slightly more users than that, so we've set up a script that runs every hour and shuts down user containers if they haven't been accessed in 24 hours. 101 | The script runs in a docker container called `cull`, so logs can be accessed via `docker logs cull`. 102 | 103 | ### Activity statistics 104 | 105 | There is another service that runs and periodically checks which users have been active recently. 106 | It then saves that information into a sqlite database, which can subsequently be downloaded and analyzed. 107 | This script runs in a docker container called `stats`, so logs can be accessed via `docker logs stats`. 108 | The sqlite database is saved to `/srv/stats/db/activity.sqlite`. 109 | 110 | ## Node servers 111 | 112 | In general, the node servers probably don't need to be accessed directly because logs for all the user containers can be accessed through swarm on the hub server. 113 | 114 | Each node server is a NFS client, with the NFS filesystem mounted at `/home`. 115 | Additionally, each node server is set up to run the singleuser IPython notebook servers in docker containers. 116 | The image is based on `jupyter/systemuser`, with the only differences being that `nbgrader` is installed so that students can validate and submit their assignments, and that terminado is installed so that users can have access to a terminal if they so desire. 117 | 118 | ## Deploying 119 | 120 | :warning: Note: you'll need to install Ansible from source (`devel` branch) to 121 | get the current versions of the Docker modules. 122 | 123 | To deploy this setup: 124 | 125 | ``` 126 | ./script/deploy 127 | ``` 128 | 129 | Note that this will stop JupyterHub if it is currently running -- so don't run 130 | this when people might be using the hub! 131 | 132 | If you need to run a particular subset of the deploy operations, you can pass a `-t` flag to specify a "tag". 133 | For example, to do all tasks relating to the statistics service, you would run `./script deploy -t stats`. 134 | These tags are defined in the tasks themselves, for example if you look at `roles/jupyterhub_host/tasks/stats.yml` you'll see that the tasks all have a "stats" tag as well as a "rebuild-tag". 135 | -------------------------------------------------------------------------------- /host_vars/jupyterhub_host: -------------------------------------------------------------------------------- 1 | $ANSIBLE_VAULT;1.1;AES256 2 | 32663566336161303364353438636561643662303362353366643831666164616634336137636333 3 | 3339613564626536656637383430353961316336333436300a623639383562353733396534613162 4 | 66366266363637316337666464363936633964356364376635376531303833316237663839343834 5 | 3936633561323266350a333161356536663061633465333537346531386633646538366562356561 6 | 65653631303530323535343061303166396333616634616234636334613336306366323531336635 7 | 63303663373366646532653139386263613062646264376231386165623732633863383961386437 8 | 34636131663931613330386661386437356164363330313436393230303335313135636366333638 9 | 62643831363739363131333763323331646361333537336336326339383134663737346634613063 10 | 39356664313564626530623066303339353431623165306161353464323334623363333339396464 11 | 39333339303239633433373937373233383332393035343030343639636335333137343134366332 12 | 61386366336666313439626361323835363166336464663532316433653862363536373039626466 13 | 35323230393265306634656666366530303333623635356366386164333938613362363362333665 14 | 35356361393138373837313932636338356234616136336336373764666431353133356562616564 15 | 30643734653731633933313935336464346564393761636235396532663431616331323430383232 16 | 33623636343030663432396639386434636136396365353963326636663939343365646539343761 17 | 32353138343566633163383939306264623536666533353866323362626362613032626238316235 18 | 64363637396633626330363965643765313662343661666132656235353062323166623635663931 19 | 61366232666166353136643663353765633032353137323263376633396334333937613063653965 20 | 65636461633863346162386163353437663330613132633362363366333130333464333965626137 21 | 36626138643566316638306561353032383134646264656139393936653731646561613539353834 22 | 32373532373462633331386365613365316365386131316461366536653331363036613665633739 23 | 66323330313566623735643833303363643737313365333430373262313765653237323736333539 24 | 34393630396439343965323432633365643837633136333037313733393961396464303030646233 25 | 63366138343163303234626461386338313566663061353135363364336239613438666561643465 26 | 33613430326465336639323964303364393561363433633461633436386261623531333664323239 27 | 30666463633836326334363963306239373937646131313337646636333136333638326334636333 28 | 39636230643163633037616135633336346661326464353630376436623963636231396462616563 29 | 33623161356235313363653832353561363834303037383063306231383436313530616330373934 30 | 33313862386439353735656432653361333236343139666535383963666138363662356131633638 31 | 62393534653538363065643831663331646265323661613438333764333939393661393363623435 32 | 64666338613865383835643264666331363866376535653165653730306463386262363539646634 33 | 38626633303630313731353736363734333335303434316133616532643330353732303961326335 34 | 39333966333734633365626161343563393730303432333061313362366433333434653564343731 35 | 30326130376138336434336339666434346337303133653261643038363733616130666133323863 36 | 30643863333938646638613461373432386366346534646432333762623061616137383762333535 37 | 31646261343061306165336137653231616362326331623135323234303730373766373964313866 38 | 37316336643633326635643263663537666665376564343639376338373563643232373762336232 39 | 39386233303037343133656630636564313962366362353634383931303839373033316230363865 40 | 36333335316363633962303730336230336163396339303963393435313936353565363534376261 41 | 39646639303336303030333462613766363630366135626636633464663164613730316631313932 42 | 64393934383838386134646663646437353038663635303839666432616532636231613433643436 43 | 39323663333335636136303563653034353232353937316532306134343430626235313732613639 44 | 37346235323465353030313333666364336461376135666365616634343537323434666431396566 45 | 30306561333764643562333662396665333362333539323264336465643631613132333238633631 46 | 62326165386531643431656565633063323662616631613762346130346534373965643034336332 47 | 64313134633538333231303163326563326265626333376538326239343732656565346435343032 48 | 34353561376364313435353038343666633538366636343730396262663138313934316538323038 49 | 33633434633063313363306163616631386161643166633634363639613864653737326338393233 50 | 39316639393939373635633238653338613665353636336435356538356137346134633363326365 51 | 65343831663038626533333636336536373033303665346231323430663062613062393634303530 52 | 64363963633162386336623362626464616564363839323736393636396638666536333436386339 53 | 34373039303630666364643364343061623038313533633934646361393238666465356335373164 54 | 33663837613136343865323032656230366637303030313265633932393939306361663763383338 55 | 30333637386664613366323838326634356562616437623363353664383063656661626633333263 56 | 32656464663264346339363635336661393836653338613438633137366339333139343935366264 57 | 39653237623736666132643666343637636537323966356230333835336533623934616430633134 58 | 64373562306539323935393039343235343834353330656131343231373933613566636230653335 59 | 61373065376533353363323839346339643262333937626465343731356337663637356163303265 60 | 33313630313633613665313831643830333764353134363533646630323237663861343335393834 61 | 37623232303837613932663333386465336437353538623735366431653261343730353230666661 62 | 61343637323161336131636338316339323438373466343439643330343763316236386639646666 63 | 31303734353365366461613139663133353265336339303137653836646630356539613430646565 64 | 39383864613765663335626464653837383130636564336331366363613965393366326231333165 65 | 35653532306265356130323638333463383239383165633533623361333537633766666561353430 66 | 64303430633963353432366666303037366361623136343965366538373062663665323563623330 67 | 63373636316533396638316331386439323933323535333436353133643032373839306335663131 68 | 63623931633664316334623466626131666635646662633537373663333036336133336334326431 69 | 62393532643133376530353465616436393661656264663936333465356237333633663137656230 70 | 39623162626437613166376565386565363166613030356234343763333137323037333733393339 71 | 61323863393233336137333530323164643031386639313563303264626562643134636632653635 72 | 30383037313735313761373330343239333836303363363862323234653239663439356365366165 73 | 33373261643866643138383132353339373030393033363930323163343661336537393862643864 74 | 63363731656132396561633361343630653235356461326232373565383030366431333032333835 75 | 39313766306435356461383165346461636236346537363539353566303866666539393738333338 76 | 36613536636434386264346633323765306137633161323131343330613364353931636462366461 77 | 37343737643163623838313066346463356261333663306230386435383238656661616330333366 78 | 65656266663665373034303436326631313136623331303932336263363364633333376265643937 79 | 35346233383964373364373239306338356463616530323133386435633163356463623463306433 80 | 61333938393264386565353938643530373238353439303464326566326266623333383566656461 81 | 62313564383462343035333463633165363366353136623839303264316364373162636365306533 82 | 66313362363939376534336364613831326662336261376163613466363361653766646264626639 83 | 31356161633739303030323862623539373061336238623462643839343837303432393631343963 84 | 63393466653233326264393737633364363531373533313163303634316635373963343737396333 85 | 33353930373731333230373832386364353463656336663562323738643565326466343238333032 86 | 34633236306132343466343336356436623964306435363562313436383636386430336666356237 87 | 61623032623661363262656366373937333466626531316466303538643662356231313731336537 88 | 66326231393234336337613163666361386263336661306362326332366635376536633664376261 89 | 38643961316564316564353434666366356135626463656463366131373539363632383333366463 90 | 64613337646431633032316161386332363935653638656166646436646338343634363366383537 91 | 66643661623736323964393438326436313537336133646262663036323531643163663965383434 92 | 36316436343032353939613733663939396535363761313166663234323434353363313662313830 93 | 65633162663630663464666564363663323163613231343935633034376339663830373835363732 94 | 63343864646433643431336638396464616262626662646130616563663630343966663834366632 95 | 63333262623132366565313335643064396632333733333864656531643734653665646239333761 96 | 30306561626338383636623735616336346265656539656336313932643963666531393235643437 97 | 31653662336137633139633038653561343838633530396534636534656134663233313966383663 98 | 37616436393130613061316462353036386262306662346562626534616631616135386633376639 99 | 34646133303963366536386161373439616332336137376466623539396630346230343366373539 100 | 30656430666663333034363033366564383865343464353163393765663136646532376463313137 101 | 34666162396432323164303864323962616636316339636130343436306336656133356139626366 102 | 64323165373061326262376664333864663763653465383034313666646463323232656531343238 103 | 37333161376461376331316539623439326165316165626466613438663430666264663937636435 104 | 61663932333030383934323136613635643861633937366137653362333435346339306363393738 105 | 37386238303238663333656563663338316232303138366561623362653563653132666337623134 106 | 65336530393065633864373936656466643661363632363832626435313335616330373864636161 107 | 32326236666231663533323031386161333665636161303136376330346634646362613663383565 108 | 61656235313766636333376139396165306237613730376466663632346236313732346463663839 109 | 32666331623439663662623364376566636238376535383664353330386562393535333539636633 110 | 34346634626636653735623666643065316165323437643637623934656664316239396463316130 111 | 61366666363965373033363963323133343034356637383738313432663166323064623766336665 112 | 62386533643434316438623363343863303065323337346330613135346437353332346162633166 113 | 31616432663139303330323335613234653462663865666266626565616166356636396666363165 114 | 36396333353839623566323666336634333134383735666138313832343832343564383835373031 115 | 37343763353461633134373237303937613532343162326165326532396134373739366133656562 116 | 61643661623662663036633063646537396263353235613537396237306635613061393538323832 117 | 31633537303236646464376132353737343734373938636235623263363137386530633733363661 118 | 66383832373034376239303437316665303039633730646165316530316531663833303537366335 119 | 37333666373761373438613862306334383136626235373136663763336432643834386566333333 120 | 61366366643439666131303130636364656232316633636537323864386432626632353536666165 121 | 39653831333565613766326339396338653266386434633363346662633866393761393232653662 122 | 38636232663164383138613935373630393930633939356564323461333836363633623432323838 123 | 62393934366163396637323931666436316161366663653833386661376133343862366661343132 124 | 34633664373063346261396438316439386532356366313031356563613338336433366338303536 125 | 35653465323934313538393538316331623965383039333536663136373931656437323735333161 126 | 62366430626461653066373030653463666537363530356439393963316632643737613736626339 127 | 62613066623330646531663534343936306635636466306537393563643133383463636432623461 128 | 64346437393530653764646461623766343432656338313138383437303763643833663837343661 129 | 35333564383561333666323034383263353362616139363965663632643533323166626362633032 130 | 30666237336635613931643361356630346662636261313263613832643036656233306231643437 131 | 33313039613862343339353836396636376337623732323236636665343138386163656237333464 132 | 32353838323130333663393630643831343034323338663462646539376364343631303731343738 133 | 30633163306662623234663234633762636565346238393263383862303466396664313039646433 134 | 61366632376261363639623164643666646162353665313831326430356338323834636162646638 135 | 32353938633866313737613235353038393035643534363336386134363665343062623562386532 136 | 30653862663532313732363135383065333666303861343630303765663735373539616633353932 137 | 30656336653162643533383666376630333661653132343530393862313361356133323430306236 138 | 32336430333331383930653136306365353864353039643361346436313632376234623338303062 139 | 62316239636566366362343733323764376564343238623734643065656534363131353832613264 140 | 31393230643164376563626362343632316632373131663532343233366334313038623031383231 141 | 37393531373730303339313261346162393034303833653232353065616436636533346631643333 142 | 37346437306132373637643361383535643438316363663862366233386637623231613466376431 143 | 32323036373432663866356430326462383630643736323230393035363866396431356431356266 144 | 31666663323135626363656239613162386235623736616365323039336339333433353536343137 145 | 37623966343862633238636663643762376333383832386634393239303034313035646366663032 146 | 61623037323765643633363666326139303332373034313138633138333037666230366365643031 147 | 35363134383965313462396432396663613065613766353762306663353539656232383965656535 148 | 61643066613530363536623034356362666666643935613234363936353162356665323266363335 149 | 32316230613934656534363332633033383834666666646362313336656630343239386235383662 150 | 38656533643538626438393937306666633062633637353064353039633734643836386231376333 151 | 33306139353166313137356264343830626266653733383235363238323165636231616634646639 152 | 36613634313663343938386235393336313935373332653463353266366437666661346434373636 153 | 66376466616463633465383838386662653838636335303863383038363530366365666666646366 154 | 61623835353139323833363366383432323763323536306266333333383863313934373762383331 155 | 36643134303061393935616234643466313439393266623532623462306364623731663836663666 156 | 64313437656361386135376538353165643261363132376533303064666636336532343833373635 157 | 61643136663331646566653662663635666238343062643032643765396433313561623063316564 158 | 64393162613135366663636565363638383036313732646439363165323437323562616139636264 159 | 38653562656332646365633461336236323462306635336565643332373631643430396463393430 160 | 38383835326261623531343164656363643035353934623732623930643831396331303762636333 161 | 35353331643565376236323261393032656431636562633535316162336533393830663539656164 162 | 31353737346564303936326639333339303232313366396533303461376361353032623163623430 163 | 37386333393536343331366463636635336165623336343365636436666138386235613133366463 164 | 64356365613462343065333632306561313933623561396230373137623162316565396238353938 165 | 38313562623961643634623336306532663738623238393936626361626430326231343935376435 166 | 33633939376438383061356130616235666234623036646439333236306362333735323135363766 167 | 38363639386166313166373034366365646636303531643664323038393463303466343665336530 168 | 61383465326530376635333836613136373031376630313839356135343163353361326563333339 169 | 31666532333162303861656465383661653839366464376331636531656362623033363161303833 170 | 34343337353334663036613764633339373963353662656438306538306136386336363663663639 171 | 38343539383436333831383861666130356566306364366562656465643861663939356335363137 172 | 65326431666133663166636536623439366130626165303738613364613961393532363638343865 173 | 34613434306434313234303361343862316138653464323335636434353635633363626130643833 174 | 35613064363164666361636663616236646530306234613632623565616163383163656632313632 175 | 36313463626663366564613932323839313364623339663566343436666639323430633937663630 176 | 65653432353561633631323135356534643835383236656534356330613334633038346462343764 177 | 35623136313432643937303161653662663934346233343363633938623566303330343436353431 178 | 65386332396437623864303035646661383738336336663462346639666333383231336662343031 179 | 31623037343861386634326430353237316231636239653236643965343931343930323736303561 180 | 30313136613336313437633462386462653632663762613266373665653462643965316636626561 181 | 37313064663266363432626134333361366635343462336564666638643033623665643137623735 182 | 34323164353831623061353631643534646636313563306365623634353331666232633235376366 183 | 31326566373962336639613939333332646433646432366232316463396133306139363933366265 184 | 39646537343836646463353933363066663032623033626130363538366536333736636435396239 185 | 36343531663239623866346335643431396538613331366136663535353261643763626332636335 186 | 36613131383362663436613563386566323163663263646162633735353932333535373933353439 187 | 63656630333936636165396638346665396265613365383332376635326430636462386234643463 188 | 38663865383539633135303733366361326334303633353837386434323866396437393863636138 189 | 62313031333163383362316536343539373037336130386533616631643933626234663039363434 190 | 61653834643133316130383832373531326238626166633039613365346163373064363533626430 191 | 33653038623363633166323764363865346332363636323032316262633430333738303861353466 192 | 37336633646161653838373734616332346232393630356634623332633764326464643136636662 193 | 66643164383838353034633634323366303665303661303939616135646437333832303535303238 194 | 37623961383238613538623163316533613664306336613663316166663330393363353261346239 195 | 31363037373134613932376135386335363433323032356633313134376533303339363036363035 196 | 30633935393438666433656431363532326435333539646233666661653765303464346433333432 197 | 32616462313163653363326462383263393335346337313166383939363433353434316363623539 198 | 30613030366233636661353166626262393565366363663637323733303835356666646361623632 199 | 37643738343762383337343935353165663666373233653762643961633764373233363336353736 200 | 66663465663166346532656534653766623061373838616439363634633434313934613535643263 201 | 61373965653831326664383861343235383062363534623961653337633136613536343530396239 202 | 39323038373136366536366236386462623535653532386662326263333636663736346631653934 203 | 65396337653634386163343165656163383831313639646439313263363132333764643963383234 204 | 33383439336435393739353865316133333634633737383532626433353061343930316638313762 205 | 36393231623563356161316164393132633166656332356439393138393761316131383930636235 206 | 33613632363430636232656332616431376339666165343861643562376462626533646362363738 207 | 34633666653938363033333937616533303233376666386365336430323033383466663366373935 208 | 63636134383031663436613733386465323835613331376139336133636633306533303532356539 209 | 36313362383632323336323761653330656661323234393061346533613861323338333234363835 210 | 38666633363632613033616233656161363132633634333138383830393261646165313163623334 211 | 31353337363666623431393339336432613730356165633732313831633863343431323939323063 212 | 30353433373137376363343837646337343866336139353865623333393437383638316463643539 213 | 39636639646537383262353437303861363537646663383236663065303630306236313839346139 214 | 64386163333036663462616638366264343239333834356663306238346234663164636237626635 215 | 38626639653261633762663265333864643264326230313138376664666639383735613964363531 216 | 66303366356364643032653533323334376433353235323365663864613564633231393539363631 217 | 38313564653036383466646531613932663465376664386132323235626161643265303866313663 218 | 6131 219 | -------------------------------------------------------------------------------- /host_vars/jupyterhub_node1: -------------------------------------------------------------------------------- 1 | $ANSIBLE_VAULT;1.1;AES256 2 | 66633730666662623263373565306264643131383939363162663965386530663435666536383536 3 | 3566353338313131656536323439616666643764653263310a303235393637336162623134613439 4 | 61306238353330323963626535333937373464306135323261313736393034303630303733333538 5 | 3462343539313162350a633835373862636432616437636333626265396337363931643364613964 6 | 66313735323861366666613362653465323336656336363530663564313062303231646262626164 7 | 38393137356564363935633965336531313335613865313661643038323339323333393839643834 8 | 62333632316639383139346161373732383837376533666464636266653663656333616363386637 9 | 32336464313534373335646131643866383637323938663531663364363237356531663935346139 10 | 38306563306264393831326165653237633431643662303761633139396236626364323730643937 11 | 66393531633037663961343633303739363632663665373362346139613433646161326662363836 12 | 32313833303364383866656331653765363739333433356137633733343735373838396363643863 13 | 34376433623965363631656366343839356639353935356438363037383666623263636166356565 14 | 61633934383232663465636665386534633931636636343661623864336462646266393162633661 15 | 66396164336163376332366134633430366239326163353664653665616136633761623438393836 16 | 30366531346432636362313433623363356133613434313238343736633163626633623161326166 17 | 34646335663564386563653932393736663239633131356363343366656463336462313864343364 18 | 61623234316231626362326564333033373338653230333462353966613334366361633838316532 19 | 63323962353466313930316238636161383665626135336161353362356130386564396364656432 20 | 34626564333232633937663539376136323832633535336534383431643938316434653163393631 21 | 63383138646431613537633061356535333937336637633361313264323164393933353362323031 22 | 34346133653937303030326163613730613562636138646633313732393032363537373835373437 23 | 61656161656236336238663765343435303736383636313430643330623966313938663236666131 24 | 39303733366232646531623435383937633765623266366362333731326463303933366136323562 25 | 34396638616364666233653531666237376137353131343835633164323263383034636237656537 26 | 61343136613432323836393262386432653931636139636164396633393763363238356262636239 27 | 30336633366565323237333835313864613061633537353737306338363237666136363831643363 28 | 62653864333061316233356132666237616139353035336533613864613134626564643065393435 29 | 38366164343731646237383664616563303030663563333432343966343035633565343736653737 30 | 61656137656164383431643037623532396431653463623731346537306132306365313039393939 31 | 62643065356231636136353365333438613935376438643732306464643134366233393039353136 32 | 38323065363633656539306166643766353430316666353062653537663263373335633361316161 33 | 31663466383362306663356533376435333634646334643864373165633464363934393766363238 34 | 66386435313764653933646661343062613131333936656162653136376138313566363632656462 35 | 64383335303763356661623930396665616535643436356136616431376135366562633934303137 36 | 36363132396462663733366664653933363139353862373861336234643930666563326366396233 37 | 63376266613930613362343936616563343430303934393763643666353666306638336432656461 38 | 33333562656437326666363637333534356535636638373735313939666463613131346133383534 39 | 64316130393439326661353834373165313238303731613530303331613535303032343733376538 40 | 35636162313538306563396263306262353336353338343836386263333364333061643839343839 41 | 64636366646334623465373835393634633566646532616465306336653162316638343230656438 42 | 38306237346130396631626536383362393430386666376264356130633663376335666564313265 43 | 38613563396533633262643630613232373863336661396365316237396432633838326261303566 44 | 30333731636234653862303065313364316165343663393932303135313538393634343736386631 45 | 30373934633735396635333964383735643139353262353166656136623764386331323731333366 46 | 36353731396661343332313135363965313466333132626364613233356137363438353564363061 47 | 36323033346437633336333731353434653439646437366138396536306661353035306336643162 48 | 31313730356637333230396134636634313561323964323831653033303432613437653663613130 49 | 62663563326439663935346632613837356134383263323734333261376537356432663336613936 50 | 62653662363962623163643237613533356164323130613331333364303861356434353564633135 51 | 63353237636133396433653035383361623834333731386362373765613330623939623665386632 52 | 33653930353335393332346138656230656561623064363533373034633432613562643631616132 53 | 30663265653436346261333065333866376333313931386130326361336531356162663864333565 54 | 34386563633233316563333564386136386638346631383362323030643835356661643161343135 55 | 37643938356563303437613634643466363463393330383130393166353730343238316137653133 56 | 63623037313034306636373231326335663834313063643835626431663439653334376565623962 57 | 66343839313733373038393263383361343436303634363561353737306538383736323564373663 58 | 61363864653138373763356664306334336534633464643733393663656564643063343330633434 59 | 36393861313062636438326333323061633835646334613764326237666466336261373732393835 60 | 66303766316332636161663635303930643066366634666632346465616330643536633161383562 61 | 66383463386331393636316331353732383263313163623234303738613236623633363664383332 62 | 38633861396664303532366434643066393837323262366164366130366166313134376339386266 63 | 37366561363861373930323963653538303464663864626432363865646237393062623234613061 64 | 35313363656438636261616138353539343264373439653135356239393330353839626565353565 65 | 36313533363461366336313738373535383463623835316638383238326264343436326233623164 66 | 35313138633930633162303332306364353038353934663065616366383365353061626131313130 67 | 34663266313432326237393766333633343731313639636363653734366364306463353833656266 68 | 39633236353539653230663331623966323131333166363533393135333066303063623332326663 69 | 64383336633731333233343939306232346132383362666235303233383134616430393065373435 70 | 35393632653431643734353833616336663835653737326638343438363030613066643031386361 71 | 34303664306137663536323230323362666534393331386233353535343165373364346664633238 72 | 31353764353565656432366335633338636661633636346433303563396634396634306539343136 73 | 36313664366339333333323262393139326165346135663939616166653562663164353035633531 74 | 61613733626331363665343631396565616464383162636632373738646132643465343663313362 75 | 37336661383437643765653231323964303236386138656162346237656561343039323735356333 76 | 31346665373833303839646565313636366563633438643834303832373834333136353466313939 77 | 63373966363535336334626536343164653362323137356130336330613231393139646266643161 78 | 65636535383737313933393135383765333466623632366633363335643963613365363036626262 79 | 37613739663864663764313235313463653166313763643131663262636339323661666131616461 80 | 38336430366263343235313234363233323631316365613739623435613935323432613835313438 81 | 63663762653366633966313537643463633732666539623666613266333632333861646530613539 82 | 30656336633631343438396161353736356365643733333132346664336437333530303834666333 83 | 66633136303837316235656639363936313633376635616231383031633636343562396366386530 84 | 39653662623232336639383833663932326633626338633064383766393330363435306432636565 85 | 31633836373030333131646130373262623938333464336336333165383035303037353561356332 86 | 66353364643462313930666462366332313865663232643264613962646266343761616234646462 87 | 63316464323364626332336539343335653666383965303134393137393265323662333934393031 88 | 65313331346363323030336237616364346232653637336663343061363637656139643564353963 89 | 39373765636139343732666133663131333031376164636165373864366234316239306162646137 90 | 37366533653665313631366366613663656165653333373830663863613138316664303336623164 91 | 39386532323733646434373665376639333765386339343061346136346566326665346462363263 92 | 38646464363764343966353165626132323238313966326639363664396334373966396133636235 93 | 39376630343534363735353538373830616534326261623861613635366666333765623734363962 94 | 62353639643930323062346533653136333736383837336433396639643265643633663161343630 95 | 63323336303765633731653465346333666662376562303734386565363561666261633433663434 96 | 34646238663562643738613730346338616664643435643131663931326330396239336165666137 97 | 33663865636565346139333561613334663138663130303030346565383037333432663538303964 98 | 64646134663831626636613733386534613133343532323764316565656537353030316362326666 99 | 33623363386461636362343163333534333364363964373435656639646136333961616162383165 100 | 39383233306639636165323261626334616630343837663336386135363462646235616464633032 101 | 35323435313634356161306136623830303539613861656363653332363162326634623162626339 102 | 39656565353937376436636433663736313861633166643263343338313434316562336262633364 103 | 66623961636566613562366537383566386431613330343066656236333231373930353962663435 104 | 35383235383936356464373134656537316461383962313031613161656330313439363766376263 105 | 64626265313862373061326334653838326566663530383863396230343930636132356261343137 106 | 34653433336333613466663730343161623161616630393632363234396431363231386233396638 107 | 37346166613736623864326262306337313234666238386335353936393632633765316439306538 108 | 64346233316435366162386635313466346332336232303233323263386165646332326332663265 109 | 38373738303161313633333863323934363565333666643234656630663066616234653263656666 110 | 37353234366533393931303832663162353434373761663532376636653039383533366663386435 111 | 35663931323238356238616330303733373466636364303931373334626634366333633461343364 112 | 38316539393438323361326536303762373634356362636531663639333133653937376132393732 113 | 30626134313530623765343839646565343239383939383639316261343765643237333231643064 114 | 33386665363462666237663037366638636335353866316562633864343339636638633534343635 115 | 35636162646361363739343335383237366162363664623132363734613662663138353836346337 116 | 36383636366636656331626665643932386533313565613562333464316664653462313337613862 117 | 35663030633236343931653562623262313762383738633061636439356134326365633935653236 118 | 38303763396239353464656338393234366536363431363134366533633761313933363561313663 119 | 39663631363465663534326163336565656231393461396436653466363834363831393263373465 120 | 30306463653236396132653133336366373731373264616332623236343835323366626138616462 121 | 36396434313835316232383965303961336366393661336136306632306236313338313866653135 122 | 31376638306262663639323437326261313664626138373039643061326662336266333837333565 123 | 39376565646565646639653139383066633534346533366137343730353938393135313532373930 124 | 65363039643664336236653333626261383434646264636235326663636237353139376539633932 125 | 31343133373339333462383966373363316566656639353363633430636530663564626231333333 126 | 34623130623464663433313936653233323162366338646332376665646264663066666139326163 127 | 33386262386163623364643337383838393733333330633337633639396637383930656665303864 128 | 62646332633864326161623736633636393236613037343135633265356161373332363836653333 129 | 66353234326337393739386633623036616336343938613466373563383930333132656333313938 130 | 61656237376264623765633661383533393030323530366536306330643465663838346333653339 131 | 39306463666131343936373537393662633564643930613365663932356561393462633739363164 132 | 33633562633135623862656264386239333638303061343163363431653465303165386534396161 133 | 32363739393634646638663331373239336433653139353033373263646138616566626637336436 134 | 34613661663063653530316366356235396463346661623362643738353436383363306531353834 135 | 64626336383832373631663735613633626164666164616462356536646432366339646137643835 136 | 30346433313863363664356433373337356234393032306432396135656638623636643261363135 137 | 36316664636631303962373664343234646633656265356638316361326135623438346139663934 138 | 62303138363839623235326362333362343139386334323737306166383763613766666331313961 139 | 30653532353231303335376631636161363966656361356139636334653438313964363838363233 140 | 30643230663064613834326337343035383231656634373933383530333364323231393436613636 141 | 34643436613666356636656366613038353731383861333166343336386361313034373665646333 142 | 66363335356166636132346532373037663565656663666661656531343862613166626339636263 143 | 64353362336335633366623334306530366431613733306266656538373538353939346631353465 144 | 62356165356464393762363866623833393433383433633036633836643364653864666363323664 145 | 33616437613162656335363135353137656234616563613939376539653133323138656534363062 146 | 37643562343563646330303931376434363737386462613962356465613934373461646430653230 147 | 65343163396630376131356434663935613034636535623430636437303463333566333161326633 148 | 63363539383033346333376465633339303862646234343333366365343562633737666364366665 149 | 37303630326562626266356430616532343232366165353834326132646261303731646565646263 150 | 37373931643832633930353865366561333366336532646439343532383331306530336331353133 151 | 31613565666462363234383537363739306138343034323031383136316132363830303362366134 152 | 36373166313430313562616436326339636230633964643438353865323364613561356263343034 153 | 32666431313362373431393163363234336135666534633833343165323664666439373465373663 154 | 39326363613236333363326339616231303962356361353239636436633130663363653937303638 155 | 36346538356232336566363166623761353333316366313535356634313464353365386461363533 156 | 35326361363130653161626637656138353934623831653137623637643165373064313839353364 157 | 38623862356237363939393639613065653230316634636263643862336166353261386163393330 158 | 65376234306361663036333936343939353837316230653636373332613736393730616234666266 159 | 39316135366539353038626533343935306463356439386565653065656264616434646463356132 160 | 30653838626539613033613366613031306137393539323633666563396664346434303036663436 161 | 65326364353130396463363762613534363262303032653933346666383637336365623061386364 162 | 37323535616332663239306534366235316665386164356336343237396639353461393064353230 163 | 38633234363466363836626662306435336332366335323332383533616366373536393834336434 164 | 61623335386462363866343735336262646634346664643066653038333465626665366233623536 165 | 64633564643366643732656232323431363262656430353430383837306335363032613731386130 166 | 61376566616638303434646263616535613935343733326230386363636336316264373338626565 167 | 65636131663936643731323062366139353361306466616165313565323365346333626161333262 168 | 61626136653637643566663131316462633063656663626532373464636162363836366462643834 169 | 62373031366533346330643363386332626135633162386338383736303137363563373664653538 170 | 35633631343930306439383931393633656334323837313463366261313938646531633433653730 171 | 34326237376135396530333639316538306635633232383764346436656564383461383231323337 172 | 38663738313031373161333562373466326666353363626332376430316130336633633163653539 173 | 66633135656636363136623964616566663237353233646365323730356339643336646532666631 174 | 61333131333366326535393461343766393365343733633534376465373662653935623337346563 175 | 64373839626633313166313566323664623631626139633263343463383834643534623765353931 176 | 63353531393363393839303731373734663363303061616330373432616638333063356236373530 177 | 36363364306166323865353933626232373237613265373732353432643335643630663138653664 178 | 64643838376538613530653132363235313937333561393461616463633665346639633134383537 179 | 65636433363833306238656565356535343865366632303530393932356436313135373837653633 180 | 31316430663139653733373965343262663436336233663066386166316332616539656539333335 181 | 31366439353436326635373363616663373835333665316163396637316638663530373132386230 182 | 66633537656438336164363663623832313836316233653233323362636634313735653161643431 183 | 63623938666465333166653930326533656164656166353534303266363133613464386634396437 184 | 65626164336530633534333631643065306132613736383637343962383964633961613137306130 185 | 34363337393435343862663034373331316664666631353639646336666337313331653062613730 186 | 36616666623063306339343230386264376365356331646464326231356334643633333738323833 187 | 64376337643166366134643133646232393930303533363236313034303362363836373938353035 188 | 36363735373061313665376365623434323566316534613163663532316130316633333437373564 189 | 63316365643532343032333235623963343238633833343438343466333161633465323334653837 190 | 36616163396335383131636537626333343565333764623230336662643363393061613332343264 191 | 38353361666262633438326431383466383834353963333233306263343137326634356437366434 192 | 65616364376135303362333839343230643464323738363464373333303434386564346363343166 193 | 30333033363937313738333837326337363230313331643863316265666533333363393462323438 194 | 33356534663230363634306136356235653565653162666135336538663665636233366635353233 195 | 33336265336130613636363639333764343062343331316533616432323835643565306439626536 196 | 64323664646133666435386635346461396232623539623834333237393638343735383733313131 197 | 66333638653662366333643365373432613863336437396539373664626433633265306532333466 198 | 32613233663735646331313861383065333863326531356433383939643938373231323531376666 199 | 66643734343863303333623135636334626663346532666263656134626363343566633833346232 200 | 38346430653833623738313031663536396666393765363438376363666532366265353763306633 201 | 61623432663237326166386538613334353338343465636231616565613663383237636537653939 202 | 33613133323939306132346662383331383761333365623938323535656234306333326636616365 203 | 36343161356635313239663038343662663930323733386136643263633132353239396637626130 204 | 31333563346262313665326163623737646165313632366661323131616266396137346564313561 205 | 32363938356366376431313466393339663737323235626263383964393436333063343665303935 206 | 64666538326439623835343831336231356133616161653962613335666139666234343739383235 207 | 63353764306461636333306432613063626335396466306438323063346539333061386137346234 208 | 39616333333537346438636630376533633135386337653331626430663234306331333362646438 209 | 39636165666562396565333230316461616663613232393166393539303832616437306338366338 210 | 37656461346139363062353366353965333535386266376661613934663630353838626232663036 211 | 64636230316631366630643937653462333035323662303934323736656333343162373163383638 212 | 39323034356636636534623436373466363663653861353265326537383964333434613665333064 213 | 65383266333334373835313137313033346462623866663462363761633334363336346465653336 214 | 34343964313833626365363463353836396332373564656535616337386230346133346435636665 215 | 66656136306535616235366566643031616261643938326331646463636561356463333862616562 216 | 34646365323636363837323430333665366431336330326564303031333165663963366361623635 217 | 30623039343766663331616539633236613532643363666138666138633839313435343333366165 218 | 3561 219 | -------------------------------------------------------------------------------- /host_vars/jupyterhub_node2: -------------------------------------------------------------------------------- 1 | $ANSIBLE_VAULT;1.1;AES256 2 | 64663166646563396262653634366637326631383433643636393263613037393835386639623765 3 | 3131303235346136633438616232333039303130356462370a623664616232366663396438353261 4 | 35613433336432333164383562356233376463393035613039333536356561393665346563613535 5 | 3139613332343264370a363062303863646438663034376532613263633462343834386131326130 6 | 61636434643630316266343164623838653962346162616232306465326162326437623532623763 7 | 38343561323135306564646137616438366137356432636433356661643966633738353763363231 8 | 31373130353533666630346566326439313335386138373937323035663336393165623030386331 9 | 66666230373065653361333137643138363961636434663232333566383438656336393837306238 10 | 30376565326664663536313039626636386363643063373363363034643532663439663534373961 11 | 32306634386262336437336261616666633230646337636436306231346635386337656661633361 12 | 63653538653966316131326439646330383731356239643530333732333063393062336530646565 13 | 33313135643161646139306465643536633532353266666236656335383534313763643362383466 14 | 39323466356561396164666464306531643734623239343364376362323935613661316265613563 15 | 61616564636234376434646230336139636631373437623838636562376661393636356263333534 16 | 62336366643265393161303234316231313834373961353634306665313635313733633336326434 17 | 35653834303766616133303537343135393165343866343931646333393136643665376336333737 18 | 37343064656439386365326665663234326262623561653933356236616430356231356630643162 19 | 36383131333834383666373130363664326562363864616232336139353963636235333337353238 20 | 35343835376562633733323165623436373066386265663337623638616533623639643266626635 21 | 38663130626337646536653162636130623662616437353839626262363133653031393865363633 22 | 38633737323565323162386361616362323365636561323263663735366135343463613763376139 23 | 64666137356537626261333466393934376262313436303235393266663966636138663638363662 24 | 39393634643962333831373263343131653039616564356330613864613363633062646636313330 25 | 66653832396364363932626239633135333838666433646238353135643563316264393634333934 26 | 65653333336361646539313237396161373435306262636438313163326231653636353466303136 27 | 34373633613634303765393138333461373566363437626133313365346463393630336135636539 28 | 65316434613862653139393765363435333734626264353738356462663036306634653739633334 29 | 38633439623536653839336661656263356261303231353838666636383534623664666439326636 30 | 31333534656337623231663062383336663466363931323565336563336562316263303338306362 31 | 32356137373065633639373839393137373362386166616364343439663563353633653062323762 32 | 62363635643330353266353665303032386665316130306337646337366533643537353237633133 33 | 36393166383034656335636263356264633066656234383936646435323133383835613231623566 34 | 37356434663364373433336535653338373535303034333432623530396231313639386234653732 35 | 66333533666338663634656232633864343934653161333263373137376636393739323765353163 36 | 39646665353336613463626430343664326232343239346466633939653830313434393365653862 37 | 64393338303439386539656434323335653835646432636261343734386233663737303130333437 38 | 30613939343830616337366437646632373131636431386234313064666439323630616266316564 39 | 64363536336438343163643733326562643438663664363437333163353238666364373963613566 40 | 63306537353336616464383030323662353766363163356465383435373666653133633630663766 41 | 38656630346330373764363039646533656331653564646664373235363333643631623436326666 42 | 32633763326164396530623261336633623961643131336437636330363863323033666563323863 43 | 35643864393062656366383861363638346638393033383264633930643332363063653538343236 44 | 65343838313033323636346233333131663839336565343864366537653831323061643339633838 45 | 32613964653237313862613039663964616435373063303265326162323365393136353334383436 46 | 31333761663834326636333239373139626431356261613765396538633737333964306435313633 47 | 38336462393732373738346430656133646630363138303063633639626362633134316430393164 48 | 39396561623238353839313466313939623166303133633932366137393739356266363034656331 49 | 39396137333030623733363633656636303033386139353237626465393630343162333764396537 50 | 33646536613761353831383339303665316361633262616539353366326466363464613238343733 51 | 61306538613639366133343262356563356263323164346330376238613036356163666232666336 52 | 61323366396333613530313366376639616563616638313933613030343237663431663635393432 53 | 63383461326435353661623235636632643339666234386261333534333633353133626435623532 54 | 31323863393338363032333036323834623232326137373166373637613965373861323062626139 55 | 39383864343166336432383766343362303439336565656437323031326562313063373432393131 56 | 32336530646338346662363831373135653665343162376665323361303364366266383035363763 57 | 35666637636163376130373636316439333861663336633130343833306530393736653365316133 58 | 30623630623730623631663734313538333530613538393133313661656133333432353366643133 59 | 36353364366465396438363338366139363735366166366530343536633765653539663966316165 60 | 31306334653464653430623734613739333635323434653961363461333335343966633863623162 61 | 35613735643639393732616239376239396564316162663036333136666366656136306239303238 62 | 66373235616163313462303230633262616535313034333234343462366233386233616262626530 63 | 63666537626664623261656338366163643634633736376433393663323633643964313936386263 64 | 31346237356138306562393839626361373064636631393437346234333331373435326331356431 65 | 32343834613933656130343336343264666464663465613330653465373532646663303134633537 66 | 34613031653263643932633130626238316630623161646536353861366264303136616134663937 67 | 39636230666332363038663336653531383636646132313138613738633932373461336230363839 68 | 37373266373431353636646638626136646438383739646435666466663836633465396535633338 69 | 33336431356639643665663062643761326433616436663266626664313434616435633733383032 70 | 66633665343137336263653138313163613265313637343037316164383436616665666664636563 71 | 34613838623939343563666432643062636665666264363539613338643866353163353663313031 72 | 37363165663731383161666438373431383963386666346539303838326539363135353066346436 73 | 65323461326331396334393836633930323638383030643730313933363263613535383530623538 74 | 39356565316362623966366533636332373366383537646236333836303634663664363030373033 75 | 37313237323164303739316536623034373336343031333536663334666663323334356630393036 76 | 36353761363233343663353334616330306635363863363266616234633532383831623438386233 77 | 65323166346332346138643665373133373330613864636432363262323962383465316265613535 78 | 64333137343030323766616239303965323030383763363138633064623339316362303535373137 79 | 30366232393935313962623836386565643237396631363664383036393236623339323038323139 80 | 33643861623361623832663638353965306563393035633162326433626637393934323065643864 81 | 66363439313431336339653738343934613466393835663661626365636365303931393239313761 82 | 33613935386333356235626631626238363732346237353863336362393431313139303537326337 83 | 31656237366231323964663765613361623733323361343364646437303934303163333864616534 84 | 37323534623438613663613462623636636665366261353133346566333030623539383335616538 85 | 31313462656562663634663237323066313861393764626139316137393831633731346433393966 86 | 35633461346138306632363063336438363839393165653361663663313137356537323437336337 87 | 64383430356565636135643265363230643833656663386166393234356464643962613635383137 88 | 64316130376632313465373661616631373265346662303333643239623763393361386663323964 89 | 33323132663331373332333366366635616537396230343737393563353834653537306663383834 90 | 32393733373936393163313463353562636337393664663133616233666238623065306365306638 91 | 63363635376165353833663631356631323061636532316337363632393862636133643836376638 92 | 64646132613230373361356239306536653466613363303136336633383865656265363432303835 93 | 36303430313261623062653530363035656362313634373765336137383136653335353735353130 94 | 37303333316262343665316263656364343830313435333265306630363364643462623265356462 95 | 31386166383839616330366337383363663337616534373536366465306139663738633362356631 96 | 39613939326362353237346662613936303631663764383633386533643831363066396566393263 97 | 36656132616235353862303637313564383238663336373333396132353563326462613163663634 98 | 37316562333238303564663732633532373561633435353939643763666334643065666163643536 99 | 38396231666637313932396361663036643230643936336636636666636437643833343131636238 100 | 36323839373230666635313662663830343937616533663461666461633032613761393836343231 101 | 31313131313761643531353562613162383637303533326431353536636435646235343934386534 102 | 36356130623538363431636465393737343565633330643033383333346538663232333265666233 103 | 37396666303836393339646434663464396231343038626663643536396562323333666462343063 104 | 30323666633730346562613436353666633030653735333032663232663665646564386437353262 105 | 35336431343530303537343633656237343566323565643638383638313266653965656462363534 106 | 66663266653535663631343038303035353866623535393162626638656632383039343534663438 107 | 30343031323366613161343562326134666439636137303464393739376134313132346439643863 108 | 39616338346562303465643065353432663264313161376131666132353461323932336161626564 109 | 66353536623737373534303639646232396637333461343231663966656236313362646437326230 110 | 30626665616165663031623165633136313365613536316334636334336632373937386638636135 111 | 65376265326566616163623761643330656637303436356231306237326335356137373534613265 112 | 35393561623134646133663037363935363231363233636437393137336131653233373534363861 113 | 62306330353463323962396630613863323665346534326364393866336561316435316538363139 114 | 33653861656138646663383139373031623137646332323632626333303337346436366337373831 115 | 62306533323137613832323632616563363735646330386462383930633332653532363136383935 116 | 66613961303139316439643363623665323865373638393136353963336264646637653932323164 117 | 38393462616662316630646335303462386632646265636635336137346536333465393861373835 118 | 31666163336137383537343535663639616661343833663237633638343166663535383666643263 119 | 31373762653531346165633661316665376130306165333937666161376133333766366265623439 120 | 61643562383164363635336630323834613061373864656433306230613731613661663434353930 121 | 64383436643134333865343461366661326532663766643563653637316639626435383765633062 122 | 62303031663737343564356233393635323838633237623736326135376334373063323535326263 123 | 66366164656432326430646661636166373039623336313461396632616261396235616532353036 124 | 63643139626335353131303933626131323831393261366536333563656139633934343439343237 125 | 61303564623031326134643931303136656330656435643962383263633166633635323763346565 126 | 62383336326132663464373737363434383364663532383030636261303037396261376136656232 127 | 63616636323832306263336534383736306561623738396462393565643333633766656237366539 128 | 65373130323732643035356166613132666266313935633564316237393763636330623831386631 129 | 66623739386436626135316231616134366163643938656635393061616333333832366437396532 130 | 33353230383163336663626463336632333431366334313064343231383030323964353532326230 131 | 38656366306434393566316261663364663463393431393233393336373639616366313136383537 132 | 34643461653630643362376633616539653933663063323733353030303439386363626239626235 133 | 62363663306561383666376335623364363136653936343464613566343037366235366232336565 134 | 32633166363735333162396538393236623533613432373936333236303064613765623136616537 135 | 31356130623061623561363562356631636337396332613133383534343563663136663534306263 136 | 66396365326336356132366136623134663330313733653465666230663165343366343466636266 137 | 62336130323663326162393363653264383632313861613138623265333232363534336161366136 138 | 32303961303537303533303930383831373536306561363236366638333765323962336434383034 139 | 39653630666531626166626162396461326664383036653932383661396564613535653162616238 140 | 64646537383733386564353433396134643434626434396265613166353965313637363162316236 141 | 38373437666466366238386337303036376438626137373463623565323938666261633135653234 142 | 33643937363465333237363434633031616633623362323061623239356431666130363739373031 143 | 37633038653039616362643632633530383864303332623362326536643664336631313234323432 144 | 61616264313261386462376565313138663461623136646339636464373532666464323136646433 145 | 38393838363535633037653137663234636637353133336464303138346566333534336230353531 146 | 38336566633835363131616530663166336561323166323931363564633936316463663533386664 147 | 32333561313334663864326664663062313865623661663366633338613033623163663339393837 148 | 34316264313638313539613532646435386663643835363664363964373331353666303737366230 149 | 65623838616331633435326635663561383363366132663933633761363131343565383137363362 150 | 38666636666439613935353363333832303238336366666664646561326166346665303139666161 151 | 62303331396336643430626663656138366561373364393235663130363530353863653138326535 152 | 30633633666361313231316362306335316636663932383437323463393738323366313533613233 153 | 63623535663861343135626236623864636434646335393664356164376263343431306161353835 154 | 30313166646531336461333438393139393131383165643161616338356139326633326339306666 155 | 38383334366633353431386663346664643362393137396636636235346635316464313262316230 156 | 63333135623936343038303463306465613966393562303264656663636562613261666137326163 157 | 65386265373235363935643766356533623666313664623831373239623939316431313031333832 158 | 61333664343361376662333038313435393861383837383366363432373963303638316165336231 159 | 35663737346161643439393265316338623239343066383061626337316339343935383839373636 160 | 32663961613234313565626239626632373061633037636435373365663665633336353962613535 161 | 61383736303263323961656335633435313931316366313165656465373565393938636165303630 162 | 31633830376132353264636562366333613238613337383164313533356364363833346237656232 163 | 66613730383337353564353031333630376231373134306366336430376466626235306237356638 164 | 37353133656138313362613434323562393235346231616334343838303334363337653131376230 165 | 61323665373161323337663636343163333061376336623963306261316539613064623163306231 166 | 36393339393630306530346364613437636531616538643563346334653334336638343730333332 167 | 37623635656234393766393430366637383637633239353638386637366137393530376536663133 168 | 66363130623333326363616665376637666636393064646636353830353837323031386234343862 169 | 31643964393432356635626266383165326239323238373137343530643531616438303235353966 170 | 30306436653961316330613134316535336364313932333861333031636635613363326539663339 171 | 61383264366333326435643262613264613466313430626337376564343734353638613338383330 172 | 65633130313334383437386265326638363064666236343738303762366361613865623434343535 173 | 34633337323863383734613038633432626666666337313932313963383135356630656462333436 174 | 39386336656131646339386532316566353530323632613866313464656635333764346139626638 175 | 30376533643538313936356264313535623134353938316338383532623737323365303938306432 176 | 36303135316561663934313336613932616361636665653137643765383438653366616537396436 177 | 31366333643432393661396363346632396237643130353731656432396137666439353364306133 178 | 34313030343039323039356161303638396261333930623437396435386363336136373432613064 179 | 64663437613533626436613739373665333736306263373934373630303932333662656133353534 180 | 36303737613566666230303364613030326137323935373132656532383737663364326634306535 181 | 34323231633538376132616631303831646632363364646636333864376364653663636436303766 182 | 30363535663730366437643436363763636232316338356366633163396665333265643833303638 183 | 36633832613032366333653565323065666131316538343866666263626634306462386335306232 184 | 36323262393334396436396633646135646535363435643933393964303637613138393561373230 185 | 32616561643635343665396130306262643133633036313861373862376234363236623739623962 186 | 34306132373632343664363564613562613139343234353531373333633734616662313666303332 187 | 65373434643732633938643036343334393866623534326239633236303235313366616664396134 188 | 37646131313439663064323662383065386438333631666634346533653666623136666563646461 189 | 36343563663632353062663635346562643636363738343837663063633431323061653331313133 190 | 64663837393331613437343661643635316364663434393864633066386161383739393761316161 191 | 62393534386366633238303030323939363438393161323365333030313563326265386535346562 192 | 65646632396366363233353234643465613631613139376630353162623562303562326134373466 193 | 66363564303332666464636630313564663137366364323831623964396461376133623739336364 194 | 62633734393861613765343836326131613536336437393334313339633036376162323633366162 195 | 30653362353834336130353439613130616132666131623531353264376239303237316437643265 196 | 62386430386432623732323662386264393932656462633434356363313361356464623333613639 197 | 35303161633733653463373439356330363232616166626363636138656338303466326231396362 198 | 64343135666265353965353564363061613632393134633565613864353361363165316334376438 199 | 64363562623339616165663838313435653936363237646539653264656461303132376432383139 200 | 37616264363138353062336335343831623464393965323366633763386165396465613565373365 201 | 31663639393065653534613039616662633532333861353436313633326366656333623531663661 202 | 36643734643862633231653033376335333063366463393233643832643235323865326661646433 203 | 65333833343733643262626265333265656534663838626163383939356139636362643030366335 204 | 64373938626531393636623230383332663866653339396463353830353065623965643464623137 205 | 65336163326633363439326362303664633136656663383865306133356532386633333561393464 206 | 39383465646562636431613563313839663933663930353838616463316533666530663533663762 207 | 62346664643134373633393434636231313236323637363662623537353434386337373330643534 208 | 66356339323639626166353734343333326263626333613265646666306566366232626236316266 209 | 31613733386432666535376364306261313262306666366136333230316362653439646361356532 210 | 39396162643332386632316264336433306365336330666565636630343564633332653961663638 211 | 65326534393935323738353137336637373039636633303936343863343163663664363338383961 212 | 35353131303338643165643134363263653534643962366661323466613465666232643034333437 213 | 31633235313062623732316465663761393761363465636637663564333639383937636365643437 214 | 37323937623430613634323662363262396332353464636339353065633032613866393665613632 215 | 38616162643833376266623730313737323638303562316534366331396334656631333834633837 216 | 64303538396532333338623836336538633535303265353033323639623465333563333766633861 217 | 63666636343832613136306332666231376361376438636330313535626235363634303265656163 218 | 6162 219 | -------------------------------------------------------------------------------- /secrets.vault.yml: -------------------------------------------------------------------------------- 1 | $ANSIBLE_VAULT;1.1;AES256 2 | 62636463323236383661373062306338393132356165386534663638616634303833663737653566 3 | 3437323561313637626337323763376432633535623430390a363062623866393637653366336538 4 | 63303130663530333938376664353831626566653337643765306164666530633136383863306232 5 | 3961363136323733310a643363663530313765663335353133623130313836326433363963333765 6 | 61333063666262653162363965393564333064656433363166343364363765346339303862333566 7 | 30633538316532306537643762653239646130656331623934633166303735643961356364326334 8 | 65393236326631636234316237666536656364363064663137393832653831663934376232313664 9 | 31613233353735313736356538643434386232346336386131326338643939323566636230616534 10 | 36376163346266646336643330626538626463633661393237663438363665383932393431633966 11 | 36376461353564316462656637346235633763343464303338666461363037333233653032313432 12 | 34376163323437666331653138376636366539366137313264623935623231356338616538646138 13 | 32626362626433383834353233393136336633316631363865326262633663666261346532656234 14 | 61653662636633386462336566336638383366373135333934623338336365636362663935323534 15 | 63306430616330363133333031623630376561616130636337356362636632356562303137623238 16 | 35383134643366333363323532323733306263663562313763376533616132623762613834623632 17 | 65656637353365323433333638656333663032386535643265306466396435333039623762613239 18 | 31316131663434366164363564373839363164346538653232663536623963363930656164323337 19 | 36323062656136323933643335356265663035643564363038346130346134363331643133373466 20 | 37306436383735653935373963393538356436373262303938306164666464376431356264613633 21 | 33633330353335313664373132383966346239356537383562663032383237666662323162326263 22 | 39643362616530613934633566656561363436323262383735366464323432613161373664613631 23 | 37383338356164306336653964383265346134353437623366616333326434336435356365343063 24 | 38316430303561373663613732373531386539653439363661313661303831613866333565393835 25 | 39643366336537333464656639373065383365626265356361633634646361353961623237366237 26 | 62346365663232333336336261386632376665613835623465666536643137376662333631663033 27 | 38346231663665323439323761313830626362373635653163656161666236353131343537306536 28 | 64643364393735643765653135613363326665663862333639356237393162336365653732333032 29 | 64633461346133363163303139366231343033623361386165616136356265646133313366323065 30 | 30393138346238653932363936356238356133376533386434633338383564663063373138643730 31 | 62333336393935333962396661396239376636306338663365343839346537646134373738623430 32 | 38643266313834653834633434363336363233616566366364393166653138353961373137316563 33 | 37393362626663386162613539323639313031353433363662656233633464663031313761653161 34 | 62303631616332633230356331363035663064353464326534663530626164376537636662343734 35 | 37613165323433333639346162306630666332323238336534323931393733336433373838313034 36 | 39613936386630333633333630383764376132633431383234343837333036616265326634623230 37 | 37373765626632653539363366393838666633326433653164633332373035353734343161656139 38 | 61633430373465633332376531666463353462326231343365353164646462356164346536623339 39 | 32333132303365323333373338633638303237333564343966336536393439373036636338653662 40 | 30383031343431383466613139373233316235343133356337313331383566366135613362656537 41 | 62313061383335643763343666633837366261363136333163663033383835393639326633383139 42 | 38313665613365613462616533646162633962613134343066373464353331616436626139323036 43 | 34356439616133613833386166303337373262383266366261393036356562363532663435656337 44 | 30636263323061333339663861653766383165326432376564393737313038386363343331383034 45 | 33663935336539663461653931373839376539326462386536393361653036303065396138313861 46 | 64313037616565636166653662613134363133353930373438666432316136373339633937303063 47 | 66383236323063613430636463653061373363666432626161323333643231663938306634306632 48 | 65306461343638623539623935626535366564383635623636303937316665633761373730353563 49 | 61633538396538333538396565643733346331323831383639643537353339376639623130643636 50 | 33323433623134363338373933623235356535353438613263353864623634663366316535373138 51 | 63343961306335393235366464363930323830306264313461363965323338333465643139613936 52 | 30303263323932396166373433383235656230306535656133316364643134653732343833373163 53 | 64323065303136373037316461343134356261643832633638373862656265656466363533376161 54 | 64326239333738363738333061326561386662623062366432356239663832343232376263313266 55 | 36363761663166613063386635623761363438613131393436626332383961623764656664626333 56 | 36616362376538613463333637376431643535383362663566363061363261653666633731316430 57 | 35393335636361373036656461626135313263623833653866666164613036613439633335303466 58 | 65383136613130626534303639396535613565376131353731323666326364633439663130313632 59 | 66346538663439666464636338356630313465386265653964666266323437386463643136613566 60 | 36386232376131633638313337623938323738613163346636366636633862303364633137383561 61 | 36323336636139383263653665313131366564306531626633636135346338366664656335623435 62 | 61626536616234343762373862316338666138343835373238633233393964643931663430623239 63 | 38353762306266393663653161633263623861643563633361623633396136613736663066356362 64 | 33363838633037343836613237393261303630323637326564393631343436336663333363336630 65 | 38623066643764356439656332636462343763396139353334356539306663376630656339373561 66 | 37613936366166636436613635316636663835616637616136613961346539316464663838376131 67 | 66356433373130313130353862353166333831393138613437623038323763363962386436353765 68 | 35323436636566306337303363653833343362373039386436656434613738303631383736653837 69 | 66393131616432393061356439306237386635653661626465343161626165363231373736363234 70 | 65616264386463323031343436356266643465313030663833383332373134626335653638306363 71 | 32386239393963343433653538616634396338623831623265373332336365323137376637333539 72 | 33366139303530663766343136376231393435653764663731363364356134393066356638373264 73 | 61633134646630313134633536333266333038343235653638633536356431666539383862376362 74 | 39653633346633363963303032643036333232306330323533373163656235613735346135653631 75 | 39393563383162323966373162303465646636323865393832313764383664326439313334656131 76 | 38393065353234376537333038643263646339326365336433336366366365663034653439353766 77 | 62333830643633383236383435363563333539613632383863306132396661643265366537373936 78 | 34313632353230656437393463306363386637343236343432613634356537393931626166343137 79 | 32633265613734343532613534366238626464613337316236383831346638656432363237633938 80 | 62653663356632616431656232616231346465393331663032643534373666383331643931653534 81 | 65633166363963633734346332326164313530363665313538306162626466653265323137353436 82 | 37326332316230353136396638383439633361373130643962626362383232323036303835386330 83 | 66376634336339656434306431393734643039343837653234666639656234343139333762366632 84 | 62643030366330383034656332663135643538643438306566363064663834323538303261303433 85 | 33653462666135623833326439316333313361653631343066336262346462333866343661313332 86 | 32383534333761363762636462656535626363613662633366643566313433646263326130363135 87 | 61373430316561333631616231373363323365323232666336363637316130373761313034303731 88 | 31383461656161623262653634633132623236316263663562656535313733633463316264363835 89 | 33653935333536346131653533666232623966636436613730623162323635396339303266356435 90 | 30373831613861306262653032356265616530303031643664613230613564636638653734363339 91 | 61653134323765663030626431643164363636343730616337363138323133636161383563653462 92 | 35353234313839623731396236393364313065363365653033626530666363663636386334353635 93 | 36383133386239613965356664313564663735343665316531303030353430393062323262356662 94 | 33343932393031623766313866626334396364353133373030386566613361373665343435336366 95 | 63663630356461646537616238326262313237363066623332373839366634326536303530353636 96 | 65656136666233643938376461623262633864386130333132623565663637613732323164373131 97 | 32333639353233356438633264353164393033646563643234313732376331393062346266363639 98 | 63313431323464393838383831366465646236343932633138626363626362386131346334643766 99 | 61363464356630336135653338356133346165386235356435643439313834313739333762636235 100 | 34353638313334373132333962346135303334313064386330646237643661653637376133613139 101 | 31623935323666366137623464306364633431663332666230616461653337393538316638396135 102 | 31383530376432303566653463653133613034343032336163333435656464393661386434323039 103 | 38316531316638346634373238393332323535643636353734313164393738383565666465373338 104 | 66396531383832643763313166343931636564383932323962306265303163633532383364366438 105 | 39626263393032636638333535633530323835396535653437356361363163383461316639326531 106 | 63656238363762646666353663616362633136636637393662636166393037366338613538313166 107 | 61326637356135313739383665356334353864663930336631623236346135343838623064333930 108 | 66353033313437663636623737323437373964663466373530306536333735613234303130356331 109 | 63643432353462333039623031343933343836386135363534376664643864613962346538373134 110 | 39633731326461643661336536343332613937636634313332396161643231343637353466623338 111 | 35643239306233633064643962656631326333666462333430623862653837656430613562306537 112 | 63653333383236653530386136346264383664343131383138333064643035363533636236626231 113 | 66626531313837643865396166353361393934303535373265363939666539366139386135656238 114 | 34643866373836656332373461633331653063373135393164353235363961303663326136346463 115 | 61643435313435373831373932303931643231333465363139373332303461386466323432666662 116 | 30303861386534363866383162306163373562373835653639643633396639343731663636653533 117 | 32373532363439366332393938663438623234356332373937333762333233366363323039383063 118 | 34663339353732636538623531383332326265623436666437303736303039396135313031666132 119 | 63653966366439663462323563376461303165633337306665383137393466653866646362303732 120 | 33643832366533366336343032316438346132363165346138386533343563343566356531316335 121 | 64383334643161333334346162393734613762633338383239366239326538363234363335626565 122 | 39306330633763616466613966373163396365326561386238323337633964393061306635393762 123 | 35363063646136373839313931656132643561643966363261303964393531613561343032656265 124 | 30316130633139643038396566313333346534363730386234323933353631353632623631613331 125 | 37313436626630643030303864343939653465383461323830643134616533333661333839396532 126 | 36616534663133643135646131376531346262333861336263623565363534363731333663326634 127 | 64396233323137653136666539343631643235303130343663633834653364613630323534336164 128 | 39313936623561656338663235623036663037626166613764336236306561643131633864396639 129 | 66376638353338346230383433656565333930303738613536333537393133643461386164663766 130 | 64386462663663386234663930323132333038326339353639353162303862396432316132643163 131 | 33306565616166616634373232646333306464383630646164633534656430343135316163666661 132 | 62373037633438336234396466623961666262333535356532613532616366626464363438306631 133 | 34646361366438623830313830343338626233343432613134386261383438613132363263386236 134 | 65633538313963326231333133396636383364363236613738646663366664396261393565656632 135 | 33633166646264373937333432326164663931383032636661343333376162353131303531386630 136 | 36626630613939313565326138393134376565393734346461643363343862613665666630303331 137 | 33313134373630653834616563356535326465646166666462633165356630633165663735656635 138 | 62653566316530383664303332356536373361626438383637613933613335656130633531383461 139 | 35623530343763646238643239393831343236626363393239623131393635366337323639313563 140 | 35353063666633343836333034386337613465353934643939376530626432333764633934316463 141 | 65346530656365663232393733643934316463643138373264333864353230363863646636666364 142 | 32313538386239386364633261373032363665666431363166646137633261386263386134313663 143 | 36613438376461356334376138346534663066333066373832363330356536343936616430366137 144 | 31393334356132396633313261373733376138623564393563643734613031613235613137376434 145 | 63336664633734363463626438626230626430353933333765303732373531353430646537653566 146 | 61326531386664303262373161626430383664393130343233313234346463643933666439356164 147 | 66643864643133663333616665356139646138323364323761323066643934306232663765633038 148 | 66363837623833646465383061303438373563643439613538346565383330353762656339316236 149 | 32323862386563316564633233333538623464613365653931353132323235656331663234326366 150 | 35383961343335396463316236353566626230616233316262396639666666306634386362646434 151 | 62646538363533636163383938373431326132353361396535373138363131336535323435343465 152 | 62393831376136646433633530663138376339666530396262336265333664633237613737613330 153 | 31653039653966613365626539636234333938323835383730313038313464376163363364663466 154 | 61636261633232323134613666306330613238323864636133383963643537623738666436616134 155 | 37363564376434366132633263643866323231356661303334373937363033343838373531646436 156 | 64323862366230393363656661643464663233663534616162643835346466396136626134646339 157 | 66626337313762346432326264636130386535323731313330383430346266663635633534623931 158 | 61343965313333663231313935323962623432363434373366663666376136633230323766623263 159 | 35623938323562386432363463343136616236383230343239383163623235366436306533386263 160 | 61613034663332613030396132623332393539396265336162313232396465303864323261306438 161 | 38666631633436363164663532316338383636323533386564386536613365393764363533636638 162 | 39616162373566316537343338333236633033333833613662323736633437616334616233613965 163 | 34393636336331353762623163363933396238323435623930623532636334656562613765343635 164 | 61353335316537616230363036393135366439353566653966313530643038333732666363616261 165 | 65333563303032626162356464633435623239636137313666646530656631623030383234336632 166 | 37323463336333336463363235636432326665326534346432643433656461623633363737616432 167 | 30623962646462363830313539356436666564306432366533613331663763383335623734643637 168 | 63343634313066383863373763396438343865373334626464643663623933356531636164376564 169 | 36356232653932363935616234616339316135666166366233363339643637363562633830316136 170 | 35363561346531386466376335323030643230613335376130303838643834633065616338313662 171 | 65326635346536303362636334643235363032663637346162323662393338643933356138646166 172 | 62646532343764373337383039383365343135643432303162623739353730653839663437356261 173 | 31316137396263383638386666373731623830326562396430303361323864333362353461633762 174 | 33333733376164623961333338323539366663653762353763656535653036313665663736376637 175 | 31623034313439303465616230366230396231656630663231363263353431386465373536393139 176 | 65666334663865663335303664383537313664653133656563303536353366396336303564383162 177 | 36386239356534386563363132316131666264653937613661613763386366343539643737643737 178 | 32353130333831313964303639366631633834343264386139343864376263353162313232333964 179 | 34613233346362616130376163633739383834386338323835396234353039396564366461633936 180 | 36376466393066663861336266376661353231626538343136646265376535613439303066326238 181 | 35396330353135663961613265326634656335653863663134353035303239643031353866613839 182 | 39346265396235666366643631633437613962653865303639386362613964303864363762666232 183 | 63313662313066656634663239343932613236343632623662636336393364626331666563633164 184 | 34626434383037326339323462323430333931663964333938613066356533306437666534326634 185 | 61313534653830393065666662303731383662623534373165623630366666666430646438376133 186 | 35656363373666643263303330663339333438363930333262633961653031353339383563633961 187 | 31666132656632333864356339323664343534353434636433663736333262323337366438353765 188 | 62366434306532363163646162336236663761313637633131356134613663353139633433613863 189 | 65326463343538636538336531376135333261623862313934333539613766356364316435366437 190 | 63313139653537303631346362623737316330336663393133383933613733646338643861353663 191 | 39316266356161343233613966323565393037393837626137366133383266376132323661376364 192 | 38363134663037306366326539393730303339383531663363366662303930393639393632653237 193 | 31346262366132333135386365636633643335656531363764343336666665353631633432363864 194 | 63636461643262393537643730313936343663313937373733373330643731323230303635393634 195 | 35353665363465623664323737636137326136353736653263643731303064626130323265366135 196 | 64663634333237653338326231376335663737363436353737623261383063323962346466643335 197 | 38303637396238643537663932366635623064636434393431383432633130373631383933373133 198 | 38396638646339373432623064306535323034333366366365623032663636336239633333613266 199 | 64623239663432383332303063626164326636666336393961366166376538396163336539663164 200 | 30343566363936653638316166373139633938353465353362343831333530626531336633613836 201 | 30366530323832333266303231623030346437306435336430366233343866343464353839656164 202 | 33336365643761333230613733316431613734616132306439363664336365373266356638383064 203 | 62376630336564623462373831633236306233653033303039313037373366373734363933363030 204 | 34663933356335353962363661656636636237663231613334636331653237353331376264323731 205 | 38613536343439386136633530643635356137643034326238343731323637306431323339383166 206 | 35313438343335303366373561616138633430633836373932653333376333306161656465386232 207 | 39353038666134623031313030323136376532613461393430363439316261386131663338316138 208 | 35376163663836343263663534616130616364383631373663653933373561613166636538613062 209 | 36336233616661326163366163363166643666613530376336333533393436396434303732343937 210 | 64326665343066333063393638316332623632636466363938323233313165626562323766303862 211 | 33383430336334393961653266633734393466343062636365626461613031373432623265306165 212 | 32333834623961383537356635306364393039393038356233336431666530343732623039376465 213 | 39323038646339336231326465616538306131383865353530316162343036613766336439343530 214 | 30646362313266373865333331656335383035616331343737326432663336376536346432616566 215 | 31323862316132303236333039653333653236653634636564373838333062303766396337656537 216 | 38343166393237303861303934333630613064323237613435393430643365633034373531386539 217 | 36646262326534666566616132366339353664363939316363626362353038396136383236396131 218 | 32313066383734323166623862303736396430353966396637323835393164613562343734313861 219 | 33333936383964653165666330636232323134653932393163653136373662386462326334633462 220 | 37303239353565383165383330393133643333346538633665333762323637613035383766363661 221 | 34333861613464636330363764623462343336623265303638613936313637306533373730303261 222 | 32353465366666643536343536313332636238303862336337616531626166613034343237386137 223 | 34373737333666626539363061623330633634643462646262613635323331613462383739313932 224 | 65636433333665666433366636313337376464363536366466386431666261623838363261646332 225 | 36343935666633646565356338313966623134373036353763333439346432623066313365336231 226 | 63303363653166636339613232356265376237663363336132366130386563383732396238373664 227 | 35353336323730343539663965656165666531346131336434636332346639666664653565333930 228 | 66626437383365326132316332656334666463623335383466643865313131323436623265663334 229 | 66316336333131633139376161333138623161306439656461343535663730316565323931623737 230 | 39346433303961373335373937306563386333376265326539653038663665666538373139656337 231 | 39343231343134636463363439373934393233323331316330343031383161636334613037653938 232 | 66636466333766393461653632376137343031303162323333356136356631643131353832383962 233 | 63633739663730383338626533353162623132323635626639393930313433316437363030313330 234 | 34663563636539396132353662373663643965653931363435643861376335323261353439336365 235 | 62366466373339343131613234626332363537303434633162653162313730376430306462633463 236 | 38663330646662613766633761303564663834343439396231393463363935373336383063383661 237 | 62326634396334303532323561663866326436343334633563373531396631653031653335373037 238 | 38343132303231346366363431313832333734393335396636363830643130663865343462373132 239 | 33373937353862343537636463663064643866306639666161383437353836386562303332643331 240 | 66383063396464323738663539343134356464663630653333363262306230666362613931393636 241 | 65306662373034373966316362393233303035613531313131366662663235313830353535646131 242 | 30343966623031383836316137303765303435643933393833383265396462373535393030303330 243 | 34336461326265333031333636363965393366633966373366363236623835333831306532343464 244 | 37376664323530383730656435653334333265303531373761633162626137633931633735356362 245 | 61616636613935303230363764356138363438643736393534326537306533343432306535613637 246 | 62353338366336636230356238643464353931393961643430303261313461336631626461323732 247 | 63303265626433623737333866653936626331623062366139373535316463343564326139646563 248 | 39343431313663306337626162623430363231333437326230613233386438636132343034613862 249 | 38646162363436373966663639636261656136373832643437656130383135373364343261383964 250 | 36343739313666613363326437323730386538626461396466623537343235633461363961326561 251 | 36623861363466393933643030636665316632373130336439393861353262333866383437613932 252 | 37363462663863346530653466633862303530353461663038376632656366656263343530383833 253 | 35376333313433623933643663313334613036656166653731303866646237663638393764393035 254 | 63336635346335353365353565643761356138373662323764666466353037363439386537646365 255 | 66376139386564303135626532623738353466336339653366303535376265313734373339613832 256 | 33356361376637346139646361303131346266303332393234373230643464626661336238393964 257 | 64393831396435343362623233613037396338643437323438613939356135353833313238306536 258 | 62306462663532663664626261313861393931363033656330646234393065323935343966333132 259 | 64646566373539363330396531326333633430613837333661323064376565616336653135353961 260 | 36663331303638376436333666396237383934626231343934626562393566356561306161656131 261 | 31623233623236656633373034643933663163393332333834633536356565353535366364303165 262 | 63636630393039346236396663326332396661636535353864323637653831396233623666633261 263 | 37346230376132626564313631306332353466643039316262613139643235333664373362396535 264 | 35666231303832323939326132373032393765666539613835336666653762613936383731316165 265 | 35303461666539366563646131396231656231373033353837633664373631343430396264313137 266 | 33353863313762306366303635386366373562346130623230366432646164626339316531633635 267 | 62653365616338376132623132333032313364323339303138376338313335633961353561376464 268 | 63323166346235346263623835386339643363376566343533356139663239383232663864613362 269 | 31313937636233623832393633353261333464363466313265393961376337303838313166313132 270 | 30366330376438653038353435336635633437656665336131613935633337393037383039636134 271 | 38623861326239303864343539333236396533313865333237636461623634306163373835386631 272 | 31393132306666323337386265373966633637373537353531633361646161663733346131383835 273 | 63666439373838623330653136323238636234323136643433373836613239343164366664396334 274 | 66626664346535353437663963363631613837666161343065653164356431623735666266613637 275 | 32353330633561653265623364616132633666646331383764666433336632393964323737333235 276 | 65303337383461636238343961633236656565643463653636313239636132333631353831373938 277 | 34353039643561383832386265396664306263303765363134313236303138626438633634666463 278 | 32663130353336653338633431346165333766333837356631626130363138346463396663613364 279 | 64356136353866636436303361313239633730343537383364373234373933373437616531663533 280 | 32356530646264396463663163623138663133646161336432663630303632386561363732623266 281 | 38383666326332666139343362353961353638646165663663356665646661313961663662393564 282 | 38626663643738646337353938373039666364326365643634306436666632356236626630643137 283 | 66316463303833373035663430303636663964656338633361363632633961623831373037356466 284 | 32333665396161636662306231626235383261643764316635316661323662383065616135656363 285 | 64353166633332343032373935646636663234323330356339343733343163613037336164383634 286 | 66396131303739383463393866396337383931356631333632623233326366346136356531626364 287 | 34346535303937323734646132616536343838363937343466653334356366326634623234623534 288 | 64356336363230373035316236356461666334373738613637386334396466643035343531383533 289 | 38323461323461326238363332653863376261613065656364623439646338623134333863306339 290 | 64626230646364653434346139636130383237313734323561663734383135333362633233363437 291 | 61333333616138323231303365343439323237666330343762356133326535306661663363383564 292 | 36333634363166626466646131613237633231366134333936356636383231316134653461313364 293 | 39383037636331303964353137623861366536663039643538366533613464326639666363663533 294 | 36663430323033373564303161306564333566663937306363356331316134646565336165346330 295 | 61393135346439653932306630646237656339356433613137333632663237616139333064306366 296 | 633661383161633161383862613532643036 297 | -------------------------------------------------------------------------------- /host_vars/proxy_server: -------------------------------------------------------------------------------- 1 | $ANSIBLE_VAULT;1.1;AES256 2 | 64646337363833653835653131663366366166336431373061323937303561373235666430343165 3 | 6331656338303033323035393964316439303537643032300a356464383139633938633761613133 4 | 38323834656235653935626166313233653039633037393566353862626535626661376639633837 5 | 3663303930663537360a613430643534333339363137623331616438633939363735646334326263 6 | 65333432666232653537373933376232306530343763373634616164626665333465316639383963 7 | 32353237316566336636386539303966613830613861613462376262623931396564616539323161 8 | 30393164313535336330326161353662653139333939636231636536643763613131353430396138 9 | 36343264656333663763343334646631313163653966633536396431336233333661373230313132 10 | 32323663626464366232333935386166616666633533613435323366303334343734363465616466 11 | 30646534343139393537376565653532653835323565346461663365623265656139313862336138 12 | 30376365363634323866623538316466396439656136646636326332366238653766356263313562 13 | 33633964333564303362626166363865666665386434613331316262353066356334363563626333 14 | 38306262616339356439663938316432633762343232326561373938613330363562623466396338 15 | 30343362646136613232356261646463323465643934333437396532376561396333343839333766 16 | 33313736633231626665666465623562336336326165313539373039373761363963356236393365 17 | 64646331313831623335343335613763613264393766626336313539373731316632626264373766 18 | 62346563343339386532393431623563343164666665643234656635333434633166376136373637 19 | 32613465333833353232656239356434373831383161303735343531356636363863626563333035 20 | 32333662646337373739323666373666626130633734303035366638363932316161646266326232 21 | 65396330343130613861623232616632643335393133363636383230343963636630393637323563 22 | 32393339376230393334386230623664346466396139656530623431336232376232383532343938 23 | 36623339326232663331333034313130306239383665663238383566366631396261323132623362 24 | 34643365373738383639623531653164373863336630306134633063343232616533623165363737 25 | 64363138353263313861393033366137383265346661663037393637343735306663663466306232 26 | 38633463306634656161616433326430316335656362323861613965646264363831363735356531 27 | 30393038316462303031613664326235353463613764303937636537376435376433326631306330 28 | 37633762313166346464393134363464326364343530333531613635356535613364386163666366 29 | 38313563396264663038316564636564393531643137393934326262363463353537363232386161 30 | 37666434653238383234303261633932646266383936393463613731346633643436636433396330 31 | 34643932633861393539346539343735363533323666666434373239663331653262613366323961 32 | 66663436396662623236663539366362613436316134353262333162343830643538336634363532 33 | 66303230323332373938633164613363666265343265313039306439386565643162623366663231 34 | 30613363396139393831363237306132393638653634386437356436313034353132343437386566 35 | 62666331386332396663636261366633303433373237323435343637343536363234323935303166 36 | 62363837666131613739653662346337656634373635613436663962396337346631383330343263 37 | 33353536646337633032323364333437363062343332616134383864363863633163383237326230 38 | 38313939656161626335373235643965653138303864396331636138633261343339383464383132 39 | 61313966613332626133623937616235386166323630663161346339376134373039626132396662 40 | 63333163666330316465363764646361303435326633346432336339623864383961616466393264 41 | 64306439616632313037393764663564613832393137336435613963363466316533373263656133 42 | 64653535663531643138613761656432306239336330666434376633373133626637643432343035 43 | 61653136373562343962356436393832613161643966363761666134343263326163333532343538 44 | 64383064643538393033353133653265303765373133643137376566366639653331373565313039 45 | 33333136653935343266353461616533633534623232356134383863616361336637623065313537 46 | 33306164356436636138656564656530383663363162366534616536656661333933616231666536 47 | 62616334343662623132316631633733633664303363366165356639313634386538303635366636 48 | 30393530333038393266373361313262663166333038316163613134666338616162663630393833 49 | 34653738303937613838353566636266653134363237356433343035616631666664643135386530 50 | 35343666646562323865323332333739626333303934356463353837386562663461646662663436 51 | 35306437333230316336383761613139373731303735646666666665623532306365323865393534 52 | 65646230633066616232323033636137636232613437643364306661626664363063323933383065 53 | 33373763353863643663653363633532313831323135333035316136343636316231396663323565 54 | 62396331633331616531656363616461323062316131386436393234313938343330383938646265 55 | 61666230393130613062396266646466316333393230656263346562396563616436663461653336 56 | 32623264633932623135396137326462383764386636376261626366633665383534343937313766 57 | 37393065396131383534386566633037393666613438646437663033356463343565333133373466 58 | 39383031383736633135653236636333383236373861393334326132353939653461666665656231 59 | 61336364656630333838326261646439663065303034633738616538383630613939626461356132 60 | 63323665383666353831636466373037366438373037313061356630363665376138323235646266 61 | 66363438383465376431666537393365383165343338303834646331316261306365326361646435 62 | 36346530333330383062643137386436366162646232343964343738343838323834303164616132 63 | 35303464386639663062353366663639323961313564343731336163616661663263613432383961 64 | 65616462666138333430353638616162383033666138376332313639333964313362656436363165 65 | 30383432626537383165633762386661393130396465346463636666626132643838316333376330 66 | 61313932393462663233393634323831353961363037613038343230663431326635383464306663 67 | 36663030363630663633323139313036643035633937643762343338376136663036363362653766 68 | 62313262613862313263616436323033373935383463636439666636336435623539316263313464 69 | 39643231303732646532356661646437376338346433663935323832393730313238366434373835 70 | 64626131303430303632336132656563356161653736306535646361623432636266636463326434 71 | 36643431373864383166336465353966383565626564346436303464343362376436363064396233 72 | 38373031633632373463373866383665396266323139646235323439333034663232376339353537 73 | 31366436316366643564383635343264663962613438643131663363306436323633363830306537 74 | 66613339313963623364373561396366633330623234663933336131363332336635313230623634 75 | 37383733653939326261633737656162396262316164303338346532633230666161373761313666 76 | 61623734303462316131386337636234646666373934343030623765356630656633653033326432 77 | 62653439383639326565336561336530383432393366353264343662316636643039653665326162 78 | 33373234313539356263343433636238623963653535396136623566313662396264643933353866 79 | 35383265376637626566316266616639336264333530616263323030636566626330383766373436 80 | 37343030613935353130623731323039393663656135333130643934636537326366623837623364 81 | 65366661386335636365363062663062323037656566393763616438356233393861623264663730 82 | 38386231383833653134613338633964383031313164326465316662623164333234623461373238 83 | 36393463343664333364303430393535393766393333356331333161343130666463326565613839 84 | 35396462303164306433636233373332396265346639343336326332343637633839363133333138 85 | 63316434326632633362653433343931646131653834373037636230646130393966316530636661 86 | 30306438346331383564353235393665376361363763643363373538336231303230386465623636 87 | 35353837346630306664366331613738623964663633313734323832623438386335326531633862 88 | 66336164353162343766646362613861303563326438323764366639616235663130333632343433 89 | 35643230393464323336333338323139663438336161643131316539383966383964333939303734 90 | 34616638643163663364636437383733363331396438646538656665306364333535313366393062 91 | 37383032663065313463333937643534396530343336626137646564343737376132343861643135 92 | 31653062646362613064653635626364396434356165303231353736306131376163626561356634 93 | 36643561303363663363663231616466353935656339666535643466643262363036383833383732 94 | 65353734393231636333396232633331326261643133613635363538343261383462396363383537 95 | 66643835343366633337316236376335316537613664386638646132383261626166613733363334 96 | 36633564363165346163313164323234346230653463616164613730383138303061343333646664 97 | 35336261616433303134343961623032336431303866306666353536376633343664393164366366 98 | 62396532373438303462326633643363336464356335313464363033356636333435643139666334 99 | 38363235363639343539616531303762633537653165613936323533376562363433333861613264 100 | 31393938623232356236653734323032386435373234366530363839643834663035303762373361 101 | 62653439383938383536666431613439313532636437313131333363373064653763613239353732 102 | 39303738356662323430393136306338663162633865356231666465353738303833343464363537 103 | 36613934343062363735353430653463356136326632623134336237623237653963333762656439 104 | 66306139626537333961313931616262353864333239393037393137343762303134643836383531 105 | 39343635656132343563663735616139336339653763353263393561326539373239376134623836 106 | 34303436316435383761363361623732316337343430303934613566303765373334346235633136 107 | 37393665333433383537316266623662373933666131356330316661323366643031366439386461 108 | 64393337363632386365316664633962613032653833333039353764653033373966373534363537 109 | 36343931633936643865353934653566653734323665393663343464633435323532353637666662 110 | 66386332383536616238613134653331353063326230343461343365393561613634316366366238 111 | 66353561353131363534636462333835646636323764353964333132623731663336303333313462 112 | 36326536623461333336323637356163373763616134313365313462646637343638396165623961 113 | 36353461616561356462633433356462356365313766653663326266643163303134306230623334 114 | 30616464363433343538353262343461326535396139333966626334313736643838393634626533 115 | 62316562316539363734646362613330643462656566666536386438393935376463383165386539 116 | 34313263343535333835306531366333306636646264393930393664663539653762633461313863 117 | 32353532663934356337313664373235663734303933646539653733393234396130383935633435 118 | 35336535393736626536653832386235653663343462373563363362363238393132346338336135 119 | 33623234356166633939613435353936393362653465333531353761393366616531633663363335 120 | 64663139613532313933343938356635633865323166343136396536316231363062653532343163 121 | 61353830623034313838313862393530393062343865303431373633323564663832393865333563 122 | 36666330373563663430616165363364666565613733326239366137653066306539336131306237 123 | 63643062616662373939663362396430613931653333343936333630343830303463363366323732 124 | 65643336663166353235643833626533323537306630366632346337326532326661656233653065 125 | 65663438303339316339383338306434623238633433343966343032636563303866393864316565 126 | 38636466616232343638653362303166323366336232316362643735613035613738383830323135 127 | 33336266353232366234343165316364343736643663396461343230363730333766303761323934 128 | 66643135646432356465383761376632386634383837356363396630663534626330373538643966 129 | 64326235346632363939306663396466383838386530626363396466633834396162623362313464 130 | 34326636323661313565613035346230633132323932306637383932306139633039346162623461 131 | 66393963653861333838366532646438326639376465656334643235626331623336643735643236 132 | 34636563323138633166393531393137623339303636306662353931613630383461636563366563 133 | 65633461336536366531353435373630353232343934313466623530623465616461626133363164 134 | 36373965313366633166636238303866303034613931333762626466343231343965333135656665 135 | 32366262376666626633336131366531626233303139616164366138636136363835663066646434 136 | 38323334353962353763323162313764616331366237393161363862303466313933643339313663 137 | 31393338336230373933356665623232653239333839626639393939303138363030346139343239 138 | 30363661396436336464386338643662323565316336316538633332323933373262386339353664 139 | 66306530323663373835376566393534303736373634613331326461613035333032333861336362 140 | 64363239386236336535356233336265643233373334366138376162363863363539363933633363 141 | 33336238316531346530383437313033663838663166306535373930663139386337326234353537 142 | 37633864636164653561636432336538346639303637653663356461646232343233303539356638 143 | 31303464343635373231353564343466353062376631356265326334366563666262653337333762 144 | 66393031616462393363666364316262353863616566353363316565613636643564393239336139 145 | 62336139666335323666396634633562383931636530373839366530663738323635346533326232 146 | 66626366303331346237313231323734616462643165386436353836323330363064336333386233 147 | 30646262353265306161653862613430336231643162646266623661306535663031323038663037 148 | 65643238666433616462303062303737386263333338633862663131313430363831396136313831 149 | 37333832336266333835346262383231616164376439656130313831656135303838663139313265 150 | 33343961376462636565353462393335636634376266383766393863313134343463613461303861 151 | 61633462343333623133653732343538303462353463363731306530313361643534623564366631 152 | 65653362373866373537343134373930323634623534653931313533633563613133643762336534 153 | 35396336643034373966626461343066393866626365386434613231336236393666643238633930 154 | 62333961323139613239323733313861323132366335356266663161393138616631383032323563 155 | 30363931376432323064663133653137393038323165383435383833636363303232383363356235 156 | 63366565616138383131653465656532316632363631316165313262353736303035343036376663 157 | 33623337363163373934393066643933333761623631616363653862316331393563623432303638 158 | 31396232623938303637323166663662656633623765343061633030633934336537376632636635 159 | 36653537613433303865346335626235616161393437343462656331323566346166356463616532 160 | 64306364363261396666353835616264623737326230343735663263643836643066613763313531 161 | 65333765653062333765383333356366636639383561313564333438616361616330333464353034 162 | 37643232333839373237613133616664326263653564656634306337363839666137623235376137 163 | 39333937653064356533383035346432646132303563333263666530313238623738616132316564 164 | 38663839326465356566356535333836333032333862316164323639326533396539663538336466 165 | 63646433353765333433323437643764663336636237303736313331616362643039666532373436 166 | 32633438393164313934396230366466353236373037636364313237313638633531333031326135 167 | 63363434363534306339323635333365343039643935333038656561383664636162363434356466 168 | 39306233653837373834313063356631663939353034663166393837333532393231666166626466 169 | 35666339653462623466623463366465306235663063626332633161333466333361363265326162 170 | 64346630396636333837656136353661323637323361326238303866376664396337623036366136 171 | 66363963383161383236346333383732336138316463393561653831623632353633303165383336 172 | 31386264393031306433643062303266323636663530626163666231373235623963386335336330 173 | 65393861313235616237316237363830323961666334633636643534393266666337373238363631 174 | 34383862303136373434356265353531393633383462643834633463363365653866653335343935 175 | 35313334313032663430313765316336663065633664393463373539663537386165373766333631 176 | 31353231383031323738333138633439646163373636613837393734353439646163393863306136 177 | 34336466363637343936353033303765343136653136653835363637333932353735633630613835 178 | 32646532343536383533373634326333373835303065356336656364316639396435643438373064 179 | 65336666306537346233346538366633323338646365373961336235633231356233393533656439 180 | 65396536313065306539616161386536356561623935613039373739623534653338313964343630 181 | 33313435653532633535623663303832656466666463353131386339666464333063316166393536 182 | 39393534623335386132613333396533343136356662306332323337623331356561313262363834 183 | 66646363346238376639316338663237663738336131333038633133393638373566613839666439 184 | 62616436363739663036666262333236383633663435303761633035623835646234386163623431 185 | 34366231633738653762643661663230376134313937613637323761356233393465396433313266 186 | 64646133353864666465623539636365343363393139303765353764383935663366313736663335 187 | 65333035333532396631303761633735663239653162613036333034363939636162306565333063 188 | 34356561663936613735666163316533363830333434306438663862643430613866303964663438 189 | 34326637376538356336323562313966316633616134383130393233333530356461653434326236 190 | 61306664373635313230373061363536326535383464666461656531313666303439343738376661 191 | 31306339623663323434383062326436313361643436663463393566313765313930363234373539 192 | 36363636363337643938316232353066356236666633373338623935663365323639613639356635 193 | 38653032663731646430396264326334613664303232386336646335396231363132663531346463 194 | 63306161616332663937396363303936376630613630353666353231396162356537663437613264 195 | 34376165316262343033636636633066376337313631643239663865663262303864343235353161 196 | 31356665646437353964316131656464613163653332623932643434646364383530646566396133 197 | 65656533623436636633346637363933323038366362356631383964653531373362343062343432 198 | 66353462623435396630356261633734646438643466666235356563343866313832393764373463 199 | 62663438663937643334646364373938616137653730643436653565333934613666383432333036 200 | 65663435323536383436656333396534623964383730343166393732633932303338303535616437 201 | 38346538353561653961393134623338363137343264633336666230663930653533306234646233 202 | 38316636623461363666393935306537316464383131356539643861373832306539656633363837 203 | 33633836383636356665646235383635663730343035366463636533333162633939326136363862 204 | 34326335373832306634316230373633316430303532306538626564643835616163623739383532 205 | 31363965633234656266623831613963316337393533646338663262303432633433326465613636 206 | 32373765623836373436353161373834633765316434316531643765653038356330633032346231 207 | 36646338666138653839353531373462663731353535336563646466316334643435633538653138 208 | 30383562326463306534633061373237666130393938376362363464346138373039383433393932 209 | 62663930636563633039613234663634383430303433313336633265356265326566346534303865 210 | 38313437313230326461653034623834313863613538393366633933363035313966333538333661 211 | 34373161353535383634376338353434393436663733653666626135346339616331343834346165 212 | 38366634373234643134373731653238373165383464616236636436643937323732656333363766 213 | 34346335306138333063343265356461373533373337383761393664353934353637643235663065 214 | 37643864343166323635393364386566386466303635353032356665653136623866386164633931 215 | 32343462663461353464353034613536343835313230613935383161623633346665373835613364 216 | 31353066393434626331396534643630343463393361316462363230663436313436646233316562 217 | 33656138613465323538303961363561653333376239346662303632613362613461666434663832 218 | 66393164626538383564363865333939366632346264336332306539643033643036643436656632 219 | 64383761323639326636663331306239333865313564383261386531393736626434653262303233 220 | 63393565356366643430613261316239336161316162373438366539326661616331386133343463 221 | 38613165666234363031653061363364663731316365313632323933346266633737643564333566 222 | 33393761633534323063363437646438643630383066333836653235366261376538366139333764 223 | 38663639323238343463643562343738326536376439343462663263663765656166363436663239 224 | 63316463376531366462626665383137333233623131666663346637316632343262393739643139 225 | 38343238393739373565346234393566626336326632326661386562313230663562663663343465 226 | 30323737363031653631616533316263316331363766316162343263633532383064656434613962 227 | 30326265346431613231373461313836623361336264343233343064393731356264383462303366 228 | 62396463666162326564393665373362643637323238333931346136663230386635353737626631 229 | 30376437306138613038613930376266343763323363343632666237643235653231313063356135 230 | 35653633613234376262393361343434663739643638363965643334623830646661373361663165 231 | 62373062353966366662303332656537366266626231323336623961356337333863366235616238 232 | 63633637383930393133393564616366313761663261366136323736393632383062356333616538 233 | 65306532373662343735383465653536306632303161363136333737643266343731316631653634 234 | 62393531363235346130376534636139396538363062646436333438643263303335613363666361 235 | 32366334653433366438393739313034646635323035386232373635636134396332373230346162 236 | 34366264396631393663633736396563386466366534323364303438613031366633663065323333 237 | 65306530373636346662643264343763653662663235376630393666646565376136663265646564 238 | 39346131323666626334303630333531363665326566373633656561653130366163336265383533 239 | 65656339343063303564393930643566366236306661343162386662613937336431633064663965 240 | 34346662313365326563643164646135633038366362303436643764623765646137613832343737 241 | 64613631613664616361353836356530643938393930313562333136636337633239633831623761 242 | 30646266353666623563666530353461623365363063366333396263336132313266636337646662 243 | 61323939346265666566626135393162623235373164383934633738303039373739633032363230 244 | 37353761326466323534666665363933396139396136653765386336363532346230363466346162 245 | 30396539643065666265663032303231613838396337313537353365623833306137636161616664 246 | 61346666376635653465613339333165373637633431346134373063323862643663316265373761 247 | 36343863306562643865326131383738643537336364633335626135383738323632623530326531 248 | 35393166653532363063303239636132643434323166616433383566623763353566623565383433 249 | 36666561316337376236613263353861303465326533633233383366333239633437616265333361 250 | 63306233303961306137633733373134373065386561346661376234646435653936636439343766 251 | 63633037396332656530303931663637343632613961386363663562313964633865393530393864 252 | 39623732366536376536656637313937363835643463333862353733396665663730353566386663 253 | 65633462353436326365323366333631376233623630313965626631373331386161333835323861 254 | 39666165663437383630363463333732663035643039303965303839616135626361376363316565 255 | 62646565333863626530313934376330343330363631633962636238343631313061356561303565 256 | 30653932363330346639333631633333383365346136663566356337333739353932363037373062 257 | 39393733356439333038383666616336376137663862633164633439366234383730636361656362 258 | 31303037343931613066636164333036343132373539646630306337346562356136343835363361 259 | 32646266383465303764363366663264623466643762343236363061396263376233643766386330 260 | 36613436383736666632623037343366343335323634376437333632313461376362366632363139 261 | 39343365393164333838343365383131363061313765346333396631393066336232313262303461 262 | 32323361663361386466613832383465373363333238363534646561363063343039383631343638 263 | 66356332306161333131383662646638613534663930646138623365303865666331383835346336 264 | 38363034623366663064613636346466333232623263636465383538373530396662663665356332 265 | 31353830313262623432366534326639326436656234656161363333653630626635313334623234 266 | 39316539623563333438616230326431653439313465346636306366333864656163656339666261 267 | 38656131393865613266373262623032656236356236306633393030313739623164666338373339 268 | 63663762386534316335356131333636393130376438393836396234343030373232393732663362 269 | 37633363636361313766366664316437643963363038663339343332353634373431376431333638 270 | 33633432383537376139656330363535336432623361613639626333613132346461613537323563 271 | 37343064636161323963623365636463363530346661326638656461643163363566303336303535 272 | 32333637323337623764346363666331393163613035353732653964316366313762316230366137 273 | 30633633623261623964613863663339386563636130383932643966363564363637386461336430 274 | 38376135643564656263303163656632646338666465356264643166616163633162633932303737 275 | 39346637363964653535633932643361626564323766646431333761343766643136313631356364 276 | 34613633346536366561643938313132363036303339656233373539363734663662396661373037 277 | 30306130363662343134396232363936353837306233636262633935366239343531333363663937 278 | 39646138373161656461386161653735356133613038366564343830383032346566373030303936 279 | 62373464386439653232626432383733616665303733613366363865323664393438366632303938 280 | 37613533356161653730333834323536663362383733373537663830316238623865663232326664 281 | 63356262313630363763313762623064306364373239616632643938633061363766666565303265 282 | 34366563666131326266623535373538323839643935376437393233643535383532373265383735 283 | 32653333643664313538363532636432353963373334386266333063366435656234623765323162 284 | 33336538383238626233366434386637363537353930653462306233306436363762393533316136 285 | 66356639376236623639376663626164336166623165316537666535633663613336346436356231 286 | 31376638346163373364313063616532386530373837643137613733393164656335366132313637 287 | 36623466363066383038646564616439313034303339623863653336383038363366346666306237 288 | 31316665393038653364666664363261653030656165356561336236373333643032643862636232 289 | 37623165623837343465623663633332383330313330666266613536633834623439613763323733 290 | 64663266366237393863353764656565646362313437653038306230643664663766373737663039 291 | 38363264643737656233323430636333383166373735336537653664303438653161653739646530 292 | 66343435653839373561343638313430386336326137356639326135326565663539323233356334 293 | 65646339353231366465316265323338303135363062353263343437333864666537376238393661 294 | 33383031386661653338646564323033633439353138346634653265643534316635366235626432 295 | 63303532303866353865313131343335306232633132396464343562623164646330323938626636 296 | 37643065333832646532303962623466653339383131656530373165363463643462323331366664 297 | 39346639656166613436663333616665346465613062623330666536656135313830353263643031 298 | 34336564663863616138386465313433646434616564646263663663303630623362613436653666 299 | 34336663633939626232666466623233653466626364353439383838633837303166303538346430 300 | 64393965356537653064346166323531383339336166363263663864393838326133616531636538 301 | 61326365623439376235396238646230336165643931303864373431666137646365363330346365 302 | 36653437633837663332396234313633623536653536666465346338623063313931316233376265 303 | 65626561353638656664306135613936323039663338313833373835313230643437303633653564 304 | 36326537626461353636353937613833343264613237386333353931636539396237643238396665 305 | 32356264646564653165656437353430623765373330646262633162353962373130313038666662 306 | 32613431626262303435626462326431323866353833363633326132386132333161643366383564 307 | 33353939343464306139343339656231623766306364623933656135643765383637333431333336 308 | 31393337323562353836663935653134636361326538396661616361313630646564653638616236 309 | 36626630633234373638613939633565346438333631653166376630396665366661393366633065 310 | 66343063643830373338373734396432393731663637653135623632376466383366643962323138 311 | 33343430386262313862303366643736663436306434336537396466303132613034343362363035 312 | 34303934623537643830633333626633336662346135343239336166336433366133643737643238 313 | 36386530373236306662346236343265653930353034316535353030333835656266626265343532 314 | 32326564316537396537343862613037633861616533646538353261366165616233643938313764 315 | 36346333323461666261646232376433303635643035646136636234633238646263376466613338 316 | 33623134353438383162323964303538643439646439363632336533333130626137363739663936 317 | 62363232336433653636376166343638613835353764393636323530636130626561353165336262 318 | 63333466623863336232323737663638373831613638333763323932613833646335373964643238 319 | 62356563393664386231643766386536396436393834333063333262636138653630356634656266 320 | 61303934633062643235353333656562646163303566613664636436363137313265356636396631 321 | 31323736626636333862323438353335636332373834393063343834653362336439353931383333 322 | 34653163353235623664613663636161643862663864353463313336336164363830346331613339 323 | 32633762356363643733323039343937646564346233663337616331626361633631653963663231 324 | 30623334653666343436346532626162616464303935653332356433653836373561376330333232 325 | 34343965336132653365393537613139623262313561313230626331653438343435623734343061 326 | 32303030313535313236616130333131616631396565373436313562636236393838373661333439 327 | 34366233656463653365333063376430646131643236323730386638373132666463613838353661 328 | 61653463633632336338336262383562626463353065386138376631383032663764326333313530 329 | 39393861363837663431333363666366363830636163343639396163346338363964383465323064 330 | 38356538346464646437633161373833343664343839643633623133366464323964623935646234 331 | 64363336353132353032666530376263396165613930636537373731623435323930613334646539 332 | 63643561323835333863376334336430323133376233336139383039393238366463663734666336 333 | 37653834633532363232663434656238383364623262373839343530666438646465633130633730 334 | 39356233353864333431356262316431643030336333633739666632343231663231333636613362 335 | 34363736366637663438396639653738623932636634373333386633653261366634633630666366 336 | 35336265663264646663333232396666623861373439326636363865646637656262306235663533 337 | 34333864323133616363626433656433343235343564613335616433646365306462666233633066 338 | 66306336643837646238333565383364376136366239356635333366346266343163326631386330 339 | 64363361333766323635323662346462353733643361633462613532623463623665376363616537 340 | 31316531616636653636666130383363633037323663353837383861663634363234326539346564 341 | 34623865303737613232336361363666303766393138393465303636353761336537393736343361 342 | 32653339386538653236353965613630656537346132393439383363626532343465393039386433 343 | 36353339333661646335346261623436396535643765353261353933333239393361353361353235 344 | 36303930666238333330613665363938376233623566656662383237323935663639626132306636 345 | 65633936356261326538616262336232353232653566336265353935343638333539363364636263 346 | 32323831383330333530316435663064396632386532363765383339633436666338343031653331 347 | 64396332356337323232613261343965316631633938306665643236646166333262343930626631 348 | 37363531303761343166623666643336383634396534316438643165323365636138616235333563 349 | 34353363306561633263333133343861306433626166316432616239316235366530326164653735 350 | 33653037636265356263333831623766366463303966626362653564336637313565626566643131 351 | 64393630313265373037366338303135626633306164306230653065616232643538303630313334 352 | 64323236376662353465306262666130636633303965336436393365303866613935643537306661 353 | 63636366623534383234303737313436373639386364333232316564346539303731356365396534 354 | 31336432383230616165333239313334333161643935663535643732343062663365356666323535 355 | 31343237653839353232353430666435633733386433383930396139313638663232633364363863 356 | 66353365393264626565633537613234616663323366396439303034623766373537653563633962 357 | 34393364343339626433383538613762656133663263333633316135653439633965383836613134 358 | 62633736663932646637343634633034616562353464306531343466336563613062636466633065 359 | 31333837633136643137333639646536323733343234313561373933383638656463336366666264 360 | 36623261343636383937623138636364626437333465333035653861626562666231363530323266 361 | 34656561663038623837326163383761363939626662626331396532346233633561376361323066 362 | 33633762386633623464373066613032343436313232336237396135373834646136306162333237 363 | 38316531343661396364393131393766343963373339316662663666386165313839393131323338 364 | 39316237633261323666393035323230356564313034656633323863336666616164356462643837 365 | 36313665666138346338646335663437376239303236646530323134396131303733313063666638 366 | 65386235636636306239653832306634613766356239333164326666363236643035316138386631 367 | 66656363373335643832313634373637326262633932323165613534326535353034303938326564 368 | 32613563303537643139336333373339363038356135636231633936643837663336373563326239 369 | 61653934343436393731356435613935616561383730353430333065333961666530623039356465 370 | 39363937323331313533386331313264396262383434353262616362313832356135383430363537 371 | 66363734343738646630313838633834396538326462336237376161343931323562646139333231 372 | 33613638323536363937643832633763646434353732646131383832366337363161663239386436 373 | 66346333376135336631353536616533643034393961326633323434366266373536323130613733 374 | 34376533633161313830653764373737396235383430623966393662633161313139636133303438 375 | 38396361373063336635656134323333306266626331663132626261666364643364343563326639 376 | 66653530353739323866383063653436616466373235336266373166643535633938323265313335 377 | 63363239663935396461353063393863363964313235373131356435303432643362616538613862 378 | 33373166653366323563656263336362643634353637336235663535323131663562643139323065 379 | 33343764626534646638623163313630393033313863616231333163613336346561366534363761 380 | 36633133336662663731303436373438623564323966383365366662663731303637643161303161 381 | 35616631663662373563643531306364303137393131353032616539663663316435303933363337 382 | 39636161666434613834393839313165336261323338643936383439386566353937316433623239 383 | 66363534346463303134663334653461343530653431613935303664613964343265356235343537 384 | 63323431626634316433636262633238323036303161303063616562633263313865306238313064 385 | 31343336663036633764643136616432316665616665643961373666643332663463396662336135 386 | 31646464363135326435623361343261306638633232326439353135366164356465366335656362 387 | 32623332656265343339383733386636336639626464313066336362393535653435616238393934 388 | 35333065666231376434363535383532353830366161636436306261663364633530336665643632 389 | 65613632623839383337613636363434363830366133646662313763353538396436376330313939 390 | 35383366336465353461306636373962303537636532353561383535353234633563383836316461 391 | 30346665633963356334616234323333666366376266643131663636383239626164346134396632 392 | 31306664653062303533633036353431396666376633373331363537303664386430373266353238 393 | 37353438613937306565383831666630353862383732376331383464633065613832393936363561 394 | 63333431656330616337383666656438363032336364663838363637363161393533613431363765 395 | 38386130306134373135636362373837373634613765626331326238613335306430306535366131 396 | 39623065623436336166366335373134646561346438396663353163366537366365653466653365 397 | 66383638613230333238666334323663326537653862313363343337323939353038623237356265 398 | 61643032366632376530626336326635353933346534386237626639363438393234 399 | --------------------------------------------------------------------------------