├── .gitattributes ├── .gitignore ├── LICENSE ├── NOTICE ├── README.md ├── docs └── images │ └── elyra-deployment-diagram.png ├── group_vars └── all.yml ├── hosts-fyre-kubernetes ├── roles ├── common │ ├── tasks │ │ ├── main.yml │ │ ├── os-packages.yml │ │ ├── ssh-config.yml │ │ └── utilities.yml │ └── templates │ │ ├── config.j2 │ │ └── do.sh.j2 ├── jupyter-enterprise-gateway │ ├── defaults │ │ └── main.yml │ ├── tasks │ │ └── main.yml │ └── templates │ │ └── enterprise-gateway-ingress-route.yaml.j2 ├── jupyterhub │ ├── defaults │ │ └── main.yml │ ├── tasks │ │ └── main.yml │ └── templates │ │ ├── jupyterhub-config.yaml.j2 │ │ ├── jupyterhub-rbac.yaml.j2 │ │ └── nfs-deployment-icp.yaml.j2 ├── kubernetes │ ├── tasks │ │ ├── kubernetes_addons.yml │ │ ├── kubernetes_common.yml │ │ ├── kubernetes_master.yml │ │ ├── kubernetes_minion.yml │ │ └── main.yml │ └── templates │ │ ├── dashboard-ingress.yaml.j2 │ │ ├── kubernetes-env.sh.j2 │ │ ├── kubernetes.repo.j2 │ │ ├── tiller-access.yaml.j2 │ │ └── traefik-dashboard-ingress-route.yaml.j2 └── spark │ ├── defaults │ └── main.yml │ ├── tasks │ ├── main.yml │ └── spark.yml │ └── templates │ └── spark-env.sh.j2 ├── setup-kubernetes.yml └── setup-spark.yml /.gitattributes: -------------------------------------------------------------------------------- 1 | # Set the default behavior to have all files normalized to Unix-style 2 | # line endings upon check-in. 3 | * text=auto 4 | 5 | # Declare files that will always have CRLF line endings on checkout. 6 | *.bat text eol=crlf 7 | 8 | # Denote all files that are truly binary and should not be modified. 9 | *.dll binary 10 | *.exp binary 11 | *.lib binary 12 | *.pdb binary 13 | *.exe binary 14 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Mac 2 | .DS_Store 3 | 4 | # Eclipse 5 | .classpath 6 | .project 7 | .settings/ 8 | .externalToolBuilders/ 9 | maven-eclipse.xml 10 | 11 | # Intellij 12 | .idea/ 13 | .idea_modules/ 14 | *.iml 15 | *.iws 16 | 17 | # Ansible temp files 18 | *.retry 19 | 20 | # Test roles and playbooks 21 | roles/test/** 22 | test.yml 23 | hosts-test 24 | hosts-elyra-fyi 25 | hosts-elyra-omg 26 | 27 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | 203 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2016-2018 Luciano Resende 2 | 3 | This product includes software developed by The Apache Software 4 | Foundation (http://www.apache.org/). 5 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Deploy a self-service Notebook environment on Kubernetes cluster 2 | 3 | This repository defines multiple [Ansible Roles](https://www.ansible.com/) to help deploying and configuring 4 | a Kubernetes cluster with [JupyterHub](https://github.com/jupyterhub/jupyterhub), 5 | [Jupyter Enterprise Gateway](https://github.com/jupyter/enterprise_gateway) and 6 | [Elyra](https://github.com/elyra-ai/elyra) extensions to 7 | [JupyterLab Notebooks](https://github.com/jupyterlab/jupyterlab). 8 | 9 | ![Deployment Diagram](./docs/images/elyra-deployment-diagram.png) 10 | 11 | # Requirements 12 | 13 | You will need a driver machine with ansible installed and a clone of the current repository: 14 | 15 | * If you are running on cloud (public/private network) 16 | * Install ansible on the edge node (with public ip) 17 | * if you are running on private cloud (public network access to all nodes) 18 | * Install ansible on your laptop and drive the deployment from it 19 | 20 | ### Installing Ansible on RHEL 21 | 22 | ``` 23 | curl -O https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm 24 | sudo rpm -i epel-release-latest-7.noarch.rpm 25 | sudo yum update -y 26 | sudo yum install -y ansible 27 | ``` 28 | 29 | ### Installing Ansible on Mac 30 | 31 | * Install Annaconda 32 | * Use pip install ansible 33 | 34 | ``` 35 | pip install --upgrade ansible 36 | ``` 37 | 38 | ### Updating Ansible configuration 39 | 40 | In order to have variable overriding from host inventory, please add the following configuration into your ~/.ansible.cfg file 41 | 42 | ``` 43 | [defaults] 44 | host_key_checking = False 45 | hash_behaviour = merge 46 | command_warnings = False 47 | ``` 48 | 49 | ### Supported/Tested Platform 50 | 51 | * RHEL 7.x 52 | * Ansible 2.10.3 53 | 54 | # Deployment 55 | 56 | ## Available deployment Ansible Roles 57 | * **common:** Basic OS updates and configurations common to all scenarios 58 | * **kubernetes:** Basic Kubernetes platform and utilities required to run either Jupyter or Spark environments 59 | * **jupyter-enterprise-gateway:** Deploy necessary images for Jupyter Enterprise Gateway and supported kernels 60 | * **jupyterhub:** Deploy JupyterHub environment using custom Notebook image for remote kernels (using Enterprise Gateway) 61 | 62 | 63 | ## Defining deployment metadata (host inventory) 64 | 65 | Ansible uses 'host inventory' files to define the cluster configuration, nodes, and groups of nodes 66 | that serves a given purpose (e.g. master node, worker nodes, etc). 67 | 68 | Below is a host inventory sample definition: 69 | 70 | ``` 71 | [all:vars] 72 | 73 | [master] 74 | lresende-kube-node-1 ansible_host=9.30.109.214 ansible_host_private=172.16.186.211 75 | 76 | [nodes] 77 | lresende-kube-node-2 ansible_host=9.30.123.95 ansible_host_private=172.16.202.136 78 | lresende-kube-node-3 ansible_host=9.30.188.28 ansible_host_private=172.16.202.144 79 | lresende-kube-node-4 ansible_host=9.30.188.36 ansible_host_private=172.16.203.8 80 | lresende-kube-node-5 ansible_host=9.30.188.38 ansible_host_private=172.16.210.6 81 | ``` 82 | 83 | ## Deployment of Kubernetes and a self-service notebook environment 84 | 85 | The sample `setup-kubernetes.yml` playbook deploys a Kubernetes cluster and configure JupyterHub 86 | and Elyra extensions to JupyterLab Notebooks. 87 | 88 | ```yaml 89 | - name: setup kubernetes 90 | hosts: all 91 | remote_user: root 92 | roles: 93 | - role: common 94 | - role: kubernetes 95 | - role: jupyter-enterprise-gateway 96 | - role: jupyterhub 97 | ``` 98 | 99 | ### Deploying 100 | 101 | ``` 102 | ansible-playbook --verbose -i 103 | ``` 104 | 105 | Example: 106 | 107 | ``` 108 | ansible-playbook --verbose setup-kubernetes.yml -c paramiko -i hosts-fyre-kubernetes 109 | ``` 110 | 111 | 112 | -------------------------------------------------------------------------------- /docs/images/elyra-deployment-diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lresende/ansible-kubernetes-cluster/7281197167f30be902b39335a1ba186bb5aab052/docs/images/elyra-deployment-diagram.png -------------------------------------------------------------------------------- /group_vars/all.yml: -------------------------------------------------------------------------------- 1 | # Account name of remote user. Ansible will use this user account to ssh into 2 | # the managed machines. The user must be able to use sudo without asking 3 | # for password. 4 | ansible_connection: ssh 5 | ansible_user: root 6 | ansible_ssh_private_key_file: ~/.ssh/ibm_rsa 7 | 8 | gather_facts: True 9 | gathering: smart 10 | 11 | # Tell ansible not to check host key 12 | host_key_checking: False 13 | 14 | # Installation folders 15 | install_temp_dir: /tmp/ansible-install 16 | install_dir: /opt 17 | 18 | # Components version (ibm 1.18.12) 19 | kube_version: 1.18.12-0 20 | kube_dashboard_version: 2.0.4 21 | 22 | # Cri-tools version 23 | #≥ 1.16.x ≥ 1.16.x master 24 | #1.15.X v1.15.0 release-1.15 25 | #1.14.X v1.14.0 release-1.14 26 | #1.13.X v1.13.0 release-1.13 27 | #1.12.X v1.12.0 release-1.12 28 | #1.11.X v1.11.1 release-1.11 29 | #1.10.X v1.0.0-beta.2 release-1.10 30 | #1.9.X v1.0.0-alpha.1 release-1.9 31 | #1.8.X v0.2 release-1.8 32 | #1.7.X v0.1 release-1.7 33 | 34 | crictl_version: v1.19.0 35 | 36 | calico_version: v3.17 37 | 38 | helm_version: 3.4.1 39 | 40 | # Kubernetes internal network for services. 41 | # Kubernetes services will get fake IP addresses from this range. 42 | # This range must not conflict with anything in your infrastructure. These 43 | # addresses do not need to be routable and must just be an unused block of space. 44 | kube_service_addresses: 10.244.0.0/16 45 | 46 | # Prefix length of kube_overlay_ip range. 47 | kube_overlay_prefix: 16 48 | 49 | # Internal DNS configuration. 50 | # Kubernetes can create and mainatain its own DNS server to resolve service names 51 | # into appropriate IP addresses. It's highly advisable to run such DNS server, 52 | # as it greatly simplifies configuration of your applications - you can use 53 | # service names instead of magic environment variables. 54 | # You still must manually configure all your containers to use this DNS server, 55 | # Kubernetes won't do this for you (yet). 56 | 57 | # Turn this varable to 'false' to disable whole DNS configuration. 58 | dns_setup: true 59 | 60 | # Internal DNS domain name. 61 | # This domain must not be used in your network. Services will be discoverable 62 | # under .., e.g. 63 | # myservice.default.kube.local 64 | dns_domain: kube.local 65 | 66 | # IP address of the DNS server. 67 | # Kubernetes will create a pod with several containers, serving as the DNS 68 | # server and expose it under this IP address. The IP address must be from 69 | # the range specified as kube_service_addresses above. 70 | # And this is the IP address you should use as address of the DNS server 71 | # in your containers. 72 | dns_server: 10.254.0.10 73 | -------------------------------------------------------------------------------- /hosts-fyre-kubernetes: -------------------------------------------------------------------------------- 1 | [master] 2 | lresende-k8-node-1 ansible_host=9.30.248.183 ansible_host_private=10.11.17.228 ansible_host_id=1 3 | 4 | [nodes] 5 | lresende-k8-node-2 ansible_host=9.30.248.231 ansible_host_private=10.11.24.198 ansible_host_id=2 6 | lresende-k8-node-3 ansible_host=9.30.253.59 ansible_host_private=10.11.28.171 ansible_host_id=3 7 | lresende-k8-node-4 ansible_host=9.30.45.150 ansible_host_private=10.11.28.206 ansible_host_id=4 8 | lresende-k8-node-5 ansible_host=9.30.218.55 ansible_host_private=10.11.29.26 ansible_host_id=5 9 | -------------------------------------------------------------------------------- /roles/common/tasks/main.yml: -------------------------------------------------------------------------------- 1 | - name: remove old install dir 2 | file: path= "{{ install_dir }}" state=absent 3 | 4 | - name: create install dir 5 | file: path="{{ install_dir }}" state=directory 6 | 7 | - name: remove old temporary install dir 8 | file: path="{{ install_temp_dir }}" state=absent 9 | 10 | - name: create temporary install dir 11 | file: path="{{ install_temp_dir }}" state=directory 12 | 13 | - import_tasks: os-packages.yml 14 | 15 | - import_tasks: utilities.yml 16 | 17 | - import_tasks: ssh-config.yml -------------------------------------------------------------------------------- /roles/common/tasks/os-packages.yml: -------------------------------------------------------------------------------- 1 | # RHEL 7.x 2 | - name: Update repository subscription on FYRE 3 | shell: 4 | cd /etc/yum.repos.d ; rm -f devit-rh7-x86_64.repo* ; wget fyreyum1.fyre.ibm.com/redhat/devit-rh7-x86_64.repo; sed -i 's/enabled = 1/enabled = 0/' /etc/yum/pluginconf.d/rhnplugin.conf 5 | when: '"fyre.ibm.com" in ansible_domain' 6 | 7 | # RHEL 8.x 8 | #- name: Update repository subscription on FYRE 9 | # shell: 10 | # cd /etc/yum.repos.d ; rm -f devit-rh8-x86_64-cd.repo* ; wget fyreyum1.fyre.ibm.com/redhat/devit-rh8-x86_64-cd.repo; sed -i 's/enabled = 1/enabled = 0/' /etc/yum/pluginconf.d/subscription-manager.conf 11 | # when: '"fyre.ibm.com" in ansible_domain' 12 | 13 | 14 | - name: install commonly used os dependencies 15 | shell: 16 | yum install -y unzip bzip2 rsync 17 | 18 | - name: upgrade all packages 19 | shell: 20 | yum update -y 21 | 22 | - name: clean yum repository 23 | shell: "yum clean all" 24 | -------------------------------------------------------------------------------- /roles/common/tasks/ssh-config.yml: -------------------------------------------------------------------------------- 1 | 2 | - name: ssh config file should be available 3 | template: src="config.j2" dest="~/.ssh/config" mode="600" 4 | 5 | -------------------------------------------------------------------------------- /roles/common/tasks/utilities.yml: -------------------------------------------------------------------------------- 1 | 2 | - name: create bin directory for utility scripts 3 | file: 4 | path: "{{ install_dir }}/bin" 5 | state: directory 6 | when: inventory_hostname in groups['master'] 7 | 8 | - name: create do.sh utility. 9 | template: 10 | src: do.sh.j2 11 | dest: "{{ install_dir }}/bin/do.sh" 12 | mode: 0744 13 | when: inventory_hostname in groups['master'] 14 | -------------------------------------------------------------------------------- /roles/common/templates/config.j2: -------------------------------------------------------------------------------- 1 | Host {% for host in groups['all'] %} {{ host }}.{{ ansible_domain }} {{ host }} {{hostvars[host].ansible_host_private}} {% endfor %} 0.0.0.0 2 | 3 | StrictHostKeyChecking no -------------------------------------------------------------------------------- /roles/common/templates/do.sh.j2: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | HOSTS=({% for host in groups['all'] %} {{ host }}.{{ ansible_domain }} {% endfor %}) 4 | 5 | for i in ${HOSTS[@]}; do 6 | ssh root@${i} "$1" 7 | done 8 | -------------------------------------------------------------------------------- /roles/jupyter-enterprise-gateway/defaults/main.yml: -------------------------------------------------------------------------------- 1 | jupyter_enterprise_gateway: 2 | version: 2.3.0 3 | image_tag: 2.3.0 4 | -------------------------------------------------------------------------------- /roles/jupyter-enterprise-gateway/tasks/main.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Make sure temp directories used by install is always available 3 | # 4 | 5 | - name: create install dir 6 | file: path="{{ install_dir }}" state=directory 7 | 8 | - name: create temporary install dir 9 | file: path="{{ install_temp_dir }}" state=directory 10 | 11 | # 12 | # Deploys Jupyter Enterprise Gateway and download kernel images 13 | # 14 | 15 | - name: Remove enterprise-gateway namespace 16 | shell: | 17 | kubectl delete namespace enterprise-gateway --force --grace-period=0 18 | when: inventory_hostname in groups['master'] 19 | ignore_errors: yes 20 | 21 | - name: download enterprise gateway and kernel images 22 | shell: | 23 | docker pull elyra/enterprise-gateway:{{ jupyter_enterprise_gateway.image_tag }} 24 | docker pull elyra/kernel-py:{{ jupyter_enterprise_gateway.image_tag }} 25 | docker pull elyra/kernel-tf-py:{{ jupyter_enterprise_gateway.image_tag }} 26 | docker pull elyra/kernel-tf-gpu-py:{{ jupyter_enterprise_gateway.image_tag }} 27 | docker pull elyra/kernel-r:{{ jupyter_enterprise_gateway.image_tag }} 28 | docker pull elyra/kernel-scala:{{ jupyter_enterprise_gateway.image_tag }} 29 | 30 | - name: download enterprise gateway yaml file 31 | block: 32 | - get_url: 33 | url: https://raw.githubusercontent.com/jupyter/enterprise_gateway/v{{ jupyter_enterprise_gateway.version }}/etc/kubernetes/enterprise-gateway.yaml 34 | dest: "{{ install_temp_dir }}/enterprise-gateway.yaml" 35 | # - replace: 36 | # path: "{{ install_temp_dir }}/enterprise-gateway.yaml" 37 | # regexp: 'elyra/enterprise-gateway:' 38 | # replace: 'elyra/enterprise-gateway:' 39 | - shell: | 40 | kubectl apply -f "{{ install_temp_dir }}/enterprise-gateway.yaml" 41 | when: inventory_hostname in groups['master'] 42 | 43 | - name: Ingress configuration for Enterprise Gateway 44 | template: 45 | src: enterprise-gateway-ingress-route.yaml.j2 46 | dest: "{{ install_temp_dir }}/enterprise-gateway-ingress-route.yaml" 47 | mode: 0644 48 | when: inventory_hostname in groups['master'] 49 | 50 | - name: Apply ingress configuration for Enterprise Gateway 51 | shell: | 52 | kubectl --kubeconfig=/etc/kubernetes/admin.conf apply -f {{ install_temp_dir }}/enterprise-gateway-ingress-route.yaml 53 | when: inventory_hostname in groups['master'] 54 | -------------------------------------------------------------------------------- /roles/jupyter-enterprise-gateway/templates/enterprise-gateway-ingress-route.yaml.j2: -------------------------------------------------------------------------------- 1 | apiVersion: traefik.containo.us/v1alpha1 2 | kind: IngressRoute 3 | metadata: 4 | namespace: enterprise-gateway 5 | name: enterprise-gateway-ingress-route 6 | spec: 7 | entryPoints: 8 | - web 9 | routes: 10 | - match: Host(`{{ groups['master'][0] }}.{{ ansible_domain }}`) && (PathPrefix(`/gateway`)) 11 | kind: Rule 12 | services: 13 | - kind: Service 14 | name: enterprise-gateway 15 | port: 8888 16 | middlewares: 17 | - name: enterprise-gateway-stripprefix 18 | 19 | --- 20 | apiVersion: traefik.containo.us/v1alpha1 21 | kind: Middleware 22 | metadata: 23 | namespace: enterprise-gateway 24 | name: enterprise-gateway-stripprefix 25 | spec: 26 | stripPrefix: 27 | prefixes: 28 | - /gateway 29 | forceSlash: true 30 | -------------------------------------------------------------------------------- /roles/jupyterhub/defaults/main.yml: -------------------------------------------------------------------------------- 1 | jupyterhub: 2 | namespace: hub 3 | helm_version: 0.10.6 # helm version coming from: https://jupyterhub.github.io/helm-chart/ 4 | home_dir: /opt/hub 5 | shared_dir: /opt/hub/shared 6 | mount_dir: /export 7 | -------------------------------------------------------------------------------- /roles/jupyterhub/tasks/main.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Make sure temp directories used by install is always available 3 | # 4 | 5 | - name: create install dir 6 | file: path="{{ install_dir }}" state=directory 7 | 8 | - name: create temporary install dir 9 | file: path="{{ install_temp_dir }}" state=directory 10 | 11 | # 12 | # Based on documentation from: 13 | # https://zero-to-jupyterhub.readthedocs.io/en/latest/ 14 | # https://zero-to-jupyterhub.readthedocs.io/en/latest/setup-jupyterhub/setup-jupyterhub.html 15 | # 16 | 17 | - name: Remove previous installation of JupyterHub 18 | shell: | 19 | helm delete hub 20 | ignore_errors: yes 21 | when: inventory_hostname in groups['master'] 22 | 23 | - name: Remove Kubernetes namespace used by JupyterHub 24 | shell: | 25 | kubectl delete namespace {{ jupyterhub.namespace }} 26 | ignore_errors: yes 27 | when: inventory_hostname in groups['master'] 28 | 29 | - name: Add JupyterHub Helm chart repository 30 | shell: | 31 | helm repo add jupyterhub https://jupyterhub.github.io/helm-chart/ 32 | helm repo update 33 | when: inventory_hostname in groups['master'] 34 | 35 | - name: Create JupyterHub secretToken 36 | command: "openssl rand -hex 32" 37 | register: temp_jupyterhub_secret_token 38 | when: inventory_hostname in groups['master'] 39 | 40 | - name: Store JupyterHub secretToken fact 41 | set_fact: jupyterhub_secret_token={{ temp_jupyterhub_secret_token.stdout }} 42 | when: inventory_hostname in groups['master'] 43 | 44 | - name: Create JupyterHub helm chart configuration 45 | template: 46 | src: jupyterhub-config.yaml.j2 47 | dest: "{{ install_temp_dir }}/jupyterhub-config.yaml" 48 | mode: 0644 49 | when: inventory_hostname in groups['master'] 50 | 51 | - name: Creates Kubernetes namespace used by JupyterHub 52 | shell: | 53 | echo -e "apiVersion: v1\nkind: Namespace\nmetadata:\n name: {{ jupyterhub.namespace }}" | kubectl apply -f - 54 | ignore_errors: yes 55 | when: inventory_hostname in groups['master'] 56 | 57 | - name: Install JupyterHub 58 | shell: | 59 | helm install hub jupyterhub/jupyterhub --namespace {{ jupyterhub.namespace }} --version {{ jupyterhub.helm_version }} --values {{ install_temp_dir }}/jupyterhub-config.yaml 60 | when: inventory_hostname in groups['master'] 61 | 62 | - name: Wait for pods to be initialized 63 | shell: | 64 | kubectl -n hub wait --for=condition=Ready --selector='app=jupyterhub' --timeout=30s pods 65 | ignore_errors: yes 66 | when: inventory_hostname in groups['master'] 67 | 68 | # - name: Remove original hub Role 69 | # shell: | 70 | # kubectl -n hub delete role hub 71 | # ignore_errors: yes 72 | # when: inventory_hostname in groups['master'] 73 | 74 | # - name: Remove original hub RoleBinding 75 | # shell: | 76 | # kubectl -n hub delete rolebinding hub 77 | # ignore_errors: yes 78 | # when: inventory_hostname in groups['master'] 79 | 80 | # # update hub service account to enable namespace creation, etc 81 | # - name: Add JupyterHub RBAC update definition 82 | # template: 83 | # src: jupyterhub-rbac.yaml.j2 84 | # dest: "{{ install_temp_dir }}/jupyterhub-rbac.yaml" 85 | # mode: 0644 86 | # when: inventory_hostname in groups['master'] 87 | 88 | # - name: Update JupyterHub RBAC definition 89 | # shell: | 90 | # kubectl apply -f {{ install_temp_dir }}/jupyterhub-rbac.yaml 91 | # when: inventory_hostname in groups['master'] 92 | 93 | - name: download singleuser image 94 | shell: | 95 | docker pull quay.io/elyra/elyra:dev 96 | 97 | # - name: download hub image 98 | # shell: | 99 | # docker rmi lresende/k8s-hub:latest 100 | # docker pull lresende/k8s-hub:latest 101 | 102 | # 103 | # Configure NFS on master 104 | # 105 | # References: 106 | # https://developer.ibm.com/code/howtos/add-nfs-provisioner-to-ICP 107 | # https://www.digitalocean.com/community/tutorials/how-to-set-up-an-nfs-mount-on-centos-6 108 | 109 | - name: stop all NFS services on all nodes 110 | shell: | 111 | chkconfig nfslock off 112 | chkconfig rpcgssd off 113 | chkconfig rpcidmapd off 114 | chkconfig portmap off 115 | chkconfig nfs off 116 | 117 | - name: Uninstall NFS on all nodes 118 | shell: | 119 | yum remove -y portmap nfs-utils nfs-utils-lib 120 | 121 | - name: install NFS on all nodes 122 | shell: | 123 | yum install -y portmap nfs-utils nfs-utils-lib 124 | 125 | - name: create hub home directory 126 | file: path="{{ jupyterhub.home_dir }}" state=directory 127 | when: inventory_hostname in groups['master'] 128 | 129 | - name: create shared NFS directory 130 | file: path="{{ jupyterhub.shared_dir }}" state=directory 131 | when: inventory_hostname in groups['master'] 132 | 133 | - name: configure NFS 134 | shell: | 135 | chkconfig nfslock on 136 | chkconfig rpcgssd on 137 | chkconfig rpcidmapd on 138 | chkconfig portmap on 139 | chkconfig nfs on 140 | service nfslock start 141 | service rpcgssd start 142 | service rpcidmapd start 143 | service portmap start 144 | service nfs start 145 | 146 | when: inventory_hostname in groups['master'] 147 | 148 | - name: Create JupyterHub NSF service descriptor 149 | template: 150 | src: nfs-deployment-icp.yaml.j2 151 | dest: "{{ install_temp_dir }}/nfs-deployment-icp.yaml" 152 | mode: 0644 153 | when: inventory_hostname in groups['master'] 154 | 155 | - name: Create NFS provisioner 156 | shell: | 157 | kubectl apply -f {{ install_temp_dir }}/nfs-deployment-icp.yaml 158 | when: inventory_hostname in groups['master'] 159 | -------------------------------------------------------------------------------- /roles/jupyterhub/templates/jupyterhub-config.yaml.j2: -------------------------------------------------------------------------------- 1 | hub: 2 | # image: 3 | # name: lresende/k8s-hub 4 | # tag: latest 5 | db: 6 | type: sqlite-memory 7 | extraConfig: |- 8 | from kubespawner import KubeSpawner 9 | from tornado import gen 10 | import yaml 11 | 12 | class CustomKubeSpawner(KubeSpawner): 13 | def get_env(self): 14 | env = super().get_env() 15 | 16 | env['KG_HTTP_USER'] = self.user.name 17 | env['KERNEL_USERNAME'] = self.user.name 18 | 19 | return env 20 | c.JupyterHub.spawner_class = CustomKubeSpawner 21 | config = '/etc/jupyter/jupyter_notebook_config.py' 22 | c.Spawner.start_timeout = 500 23 | # c.KubeSpawner.namespace_name_template = 'hub-{username}' 24 | proxy: 25 | secretToken: "{{ jupyterhub_secret_token }}" 26 | service: 27 | type: NodePort 28 | 29 | auth: 30 | admin: 31 | users: 32 | - root 33 | - jovyan 34 | 35 | ingress: 36 | enabled: true 37 | annotations: 38 | kubernetes.io/ingress.class: traefik 39 | hosts: 40 | - {{ groups['master'][0] }}.{{ ansible_domain }} 41 | 42 | singleuser: 43 | defaultUrl: "/lab" 44 | image: 45 | name: quay.io/elyra/elyra 46 | # change to a specific release version as appropriated 47 | tag: dev 48 | # disable this in a production environment 49 | pullPolicy: "Always" 50 | storage: 51 | dynamic: 52 | storageClass: nfs-dynamic 53 | extraEnv: 54 | JUPYTERHUB_SINGLEUSER_APP: "jupyter_server.serverapp.ServerApp" 55 | JUPYTER_GATEWAY_URL: http://{{ groups['master'][0] }}.{{ ansible_domain }}/gateway 56 | JUPYTER_GATEWAY_REQUEST_TIMEOUT: "120" 57 | 58 | rbac: 59 | enabled: true 60 | 61 | debug: 62 | enabled: true 63 | -------------------------------------------------------------------------------- /roles/jupyterhub/templates/jupyterhub-rbac.yaml.j2: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1beta1 2 | kind: ClusterRole 3 | metadata: 4 | name: hub 5 | labels: 6 | app: jupyterhub 7 | chart: jupyterhub-0.8.0 8 | component: hub 9 | heritage: Tiller 10 | release: hub 11 | rules: 12 | - apiGroups: [""] 13 | resources: ["pods", "persistentvolumnes", "persistentvolumeclaims"] 14 | verbs: ["get", "watch", "list", "create", "delete"] 15 | - apiGroups: [""] 16 | resources: ["events"] 17 | verbs: ["get", "watch", "list"] 18 | - apiGroups: [""] 19 | resources: ["namespaces", "serviceaccounts"] 20 | verbs: ["get", "watch", "create", "list"] 21 | - apiGroups: ["rbac.authorization.k8s.io"] 22 | resources: ["roles", "rolebindings"] 23 | verbs: ["get", "list", "create", "delete"] 24 | --- 25 | apiVersion: rbac.authorization.k8s.io/v1beta1 26 | kind: ClusterRoleBinding 27 | metadata: 28 | name: hub 29 | labels: 30 | app: jupyterhub 31 | chart: jupyterhub-0.8.0 32 | component: hub 33 | heritage: Tiller 34 | release: hub 35 | subjects: 36 | - kind: ServiceAccount 37 | name: hub 38 | namespace: hub 39 | roleRef: 40 | kind: ClusterRole 41 | name: hub 42 | apiGroup: rbac.authorization.k8s.io 43 | -------------------------------------------------------------------------------- /roles/jupyterhub/templates/nfs-deployment-icp.yaml.j2: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | name: nfs-provisioner 5 | namespace: {{ jupyterhub.namespace }} 6 | --- 7 | kind: ClusterRole 8 | apiVersion: rbac.authorization.k8s.io/v1 9 | metadata: 10 | name: nfs-provisioner-runner 11 | rules: 12 | - apiGroups: [""] 13 | resources: ["persistentvolumes"] 14 | verbs: ["get", "list", "watch", "create", "delete"] 15 | - apiGroups: [""] 16 | resources: ["persistentvolumeclaims"] 17 | verbs: ["get", "list", "watch", "update"] 18 | - apiGroups: ["storage.k8s.io"] 19 | resources: ["storageclasses"] 20 | verbs: ["get", "list", "watch"] 21 | - apiGroups: ["storage.k8s.io"] 22 | resources: ["volumeattachments"] 23 | verbs: ["get", "list", "watch", "update"] 24 | - apiGroups: [""] 25 | resources: ["events"] 26 | verbs: ["create", "update", "patch"] 27 | - apiGroups: [""] 28 | resources: ["services", "endpoints"] 29 | verbs: ["get"] 30 | - apiGroups: ["extensions"] 31 | resources: ["podsecuritypolicies"] 32 | resourceNames: ["nfs-provisioner"] 33 | verbs: ["use"] 34 | --- 35 | kind: ClusterRoleBinding 36 | apiVersion: rbac.authorization.k8s.io/v1 37 | metadata: 38 | name: run-nfs-provisioner 39 | subjects: 40 | - kind: ServiceAccount 41 | name: nfs-provisioner 42 | namespace: {{ jupyterhub.namespace }} 43 | roleRef: 44 | kind: ClusterRole 45 | name: nfs-provisioner-runner 46 | apiGroup: rbac.authorization.k8s.io 47 | --- 48 | kind: Role 49 | apiVersion: rbac.authorization.k8s.io/v1 50 | metadata: 51 | name: leader-locking-nfs-provisioner 52 | namespace: {{ jupyterhub.namespace }} 53 | rules: 54 | - apiGroups: [""] 55 | resources: ["endpoints"] 56 | verbs: ["get", "list", "watch", "create", "update", "patch"] 57 | --- 58 | kind: RoleBinding 59 | apiVersion: rbac.authorization.k8s.io/v1 60 | metadata: 61 | name: leader-locking-nfs-provisioner 62 | namespace: {{ jupyterhub.namespace }} 63 | subjects: 64 | - kind: ServiceAccount 65 | name: nfs-provisioner 66 | namespace: {{ jupyterhub.namespace }} 67 | roleRef: 68 | kind: Role 69 | name: leader-locking-nfs-provisioner 70 | apiGroup: rbac.authorization.k8s.io 71 | --- 72 | kind: Service 73 | apiVersion: v1 74 | metadata: 75 | name: nfs-provisioner 76 | namespace: {{ jupyterhub.namespace }} 77 | labels: 78 | app: nfs-provisioner 79 | spec: 80 | ports: 81 | - name: rpcbind 82 | port: 111 83 | - name: rpcbind-udp 84 | port: 111 85 | protocol: UDP 86 | - name: statd 87 | port: 662 88 | - name: statd-udp 89 | port: 662 90 | protocol: UDP 91 | - name: rquotad 92 | port: 875 93 | - name: rquotad-udp 94 | port: 875 95 | protocol: UDP 96 | - name: nfs 97 | port: 2049 98 | - name: nfs-udp 99 | port: 2049 100 | protocol: UDP 101 | - name: mountd 102 | port: 20048 103 | - name: mountd-udp 104 | port: 20048 105 | protocol: UDP 106 | - name: nlockmgr 107 | port: 32803 108 | - name: nlockmgr-udp 109 | port: 32803 110 | protocol: UDP 111 | selector: 112 | app: nfs-provisioner 113 | --- 114 | kind: Deployment 115 | apiVersion: apps/v1 116 | metadata: 117 | name: nfs-provisioner 118 | namespace: {{ jupyterhub.namespace }} 119 | spec: 120 | replicas: 1 121 | selector: 122 | matchLabels: 123 | app: nfs-provisioner 124 | strategy: 125 | type: Recreate 126 | template: 127 | metadata: 128 | labels: 129 | app: nfs-provisioner 130 | spec: 131 | # The following toleration and nodeSelector will place the nfs provisioner on the ICP master node 132 | tolerations: 133 | - key: "node-role.kubernetes.io/master" 134 | operator: "Exists" 135 | effect: "NoSchedule" 136 | nodeSelector: 137 | kubernetes.io/hostname: {{ groups['master'][0] }}.{{ ansible_domain }} 138 | serviceAccount: nfs-provisioner 139 | containers: 140 | - name: nfs-provisioner 141 | image: quay.io/kubernetes_incubator/nfs-provisioner:latest 142 | ports: 143 | - name: rpcbind 144 | containerPort: 111 145 | - name: rpcbind-udp 146 | containerPort: 111 147 | protocol: UDP 148 | - name: statd 149 | containerPort: 662 150 | - name: statd-udp 151 | containerPort: 662 152 | protocol: UDP 153 | - name: rquotad 154 | containerPort: 875 155 | - name: rquotad-udp 156 | containerPort: 875 157 | protocol: UDP 158 | - name: nfs 159 | containerPort: 2049 160 | - name: nfs-udp 161 | containerPort: 2049 162 | protocol: UDP 163 | - name: mountd 164 | containerPort: 20048 165 | - name: mountd-udp 166 | containerPort: 20048 167 | protocol: UDP 168 | - name: nlockmgr 169 | containerPort: 32803 170 | - name: nlockmgr-udp 171 | containerPort: 32803 172 | protocol: UDP 173 | securityContext: 174 | capabilities: 175 | add: 176 | - DAC_READ_SEARCH 177 | - SYS_RESOURCE 178 | args: 179 | - "-provisioner=example.com/nfs" 180 | env: 181 | - name: POD_IP 182 | valueFrom: 183 | fieldRef: 184 | fieldPath: status.podIP 185 | - name: SERVICE_NAME 186 | value: nfs-provisioner 187 | - name: POD_NAMESPACE 188 | valueFrom: 189 | fieldRef: 190 | fieldPath: metadata.namespace 191 | imagePullPolicy: "IfNotPresent" 192 | volumeMounts: 193 | - name: export-volume 194 | mountPath: {{ jupyterhub.mount_dir }} 195 | volumes: 196 | - name: export-volume 197 | hostPath: 198 | path: {{ jupyterhub.shared_dir }} 199 | --- 200 | kind: StorageClass 201 | apiVersion: storage.k8s.io/v1 202 | metadata: 203 | name: nfs-dynamic 204 | namespace: {{ jupyterhub.namespace }} 205 | annotations: 206 | storageclass.kubernetes.io/is-default-class: "true" 207 | provisioner: example.com/nfs 208 | -------------------------------------------------------------------------------- /roles/kubernetes/tasks/kubernetes_addons.yml: -------------------------------------------------------------------------------- 1 | 2 | # https://docs.projectcalico.org/v3.0/getting-started/kubernetes/installation/hosted/kubeadm/#requirements 3 | # https://docs.projectcalico.org/getting-started/kubernetes/self-managed-onprem/onpremises 4 | - name: Install Calico Network 5 | shell: | 6 | kubectl --kubeconfig=/etc/kubernetes/admin.conf apply -f https://docs.projectcalico.org/{{ calico_version }}/manifests/calico.yaml 7 | 8 | # Install Helm 3.x 9 | - name: Install Helm 10 | shell: | 11 | rm -rf /opt/helm 12 | rm -f /usr/local/bin/helm 13 | rm -f /usr/local/bin/tiller 14 | wget https://get.helm.sh/helm-v{{ helm_version }}-linux-amd64.tar.gz 15 | mkdir /opt/helm 16 | tar -xvf helm-v{{ helm_version }}-linux-amd64.tar.gz --strip 1 --directory /opt/helm 17 | rm helm-v{{ helm_version }}-linux-amd64.tar.gz 18 | chmod u+x /opt/helm/helm 19 | ln -s /opt/helm/helm /usr/local/bin/helm 20 | 21 | # Install Traefik using Helm 22 | - name: Add Traefik Helm chart repository 23 | shell: | 24 | helm repo add traefik https://helm.traefik.io/traefik 25 | helm repo update 26 | 27 | - name: Install Traefik 28 | shell: | 29 | helm install traefik traefik/traefik 30 | 31 | - name: Add Traefik Dashboard ingress 32 | template: 33 | src: traefik-dashboard-ingress-route.yaml.j2 34 | dest: "{{ install_temp_dir }}/traefik-dashboard-ingress-route.yaml" 35 | mode: 0644 36 | 37 | - name: Wait for traefik to be ready 38 | pause: 39 | minutes: 1 40 | 41 | - name: service content 42 | shell: | 43 | kubectl get svc/traefik -o yaml 44 | register: temp_service_content 45 | 46 | - name: Update Traefik service with clusterIP 47 | shell: | 48 | kubectl get svc/traefik -o yaml | sed -e "s/^spec:/spec:\n externalIPs:\n - {{ hostvars[groups['master'][0]].ansible_host }}/" | kubectl apply -f - 49 | 50 | - name: Install Traefik Ingress Controller 51 | shell: | 52 | kubectl apply -f {{ install_temp_dir }}/traefik-dashboard-ingress-route.yaml 53 | 54 | # Reference - https://kubernetes.io/docs/tasks/access-application-cluster/web-ui-dashboard/ 55 | # Use the following for retrieving a bearer token 56 | # kubectl -n kubernetes-dashboard describe secret $(kubectl -n kubernetes-dashboard get secret | grep admin-user | awk '{print $1}') 57 | 58 | - name: Install Dashboard 59 | shell: | 60 | kubectl --kubeconfig=/etc/kubernetes/admin.conf apply -f https://raw.githubusercontent.com/kubernetes/dashboard/v{{kube_dashboard_version}}/aio/deploy/alternative.yaml 61 | # kubectl --kubeconfig=/etc/kubernetes/admin.conf apply -f https://raw.githubusercontent.com/kubernetes/dashboard/v{{kube_dashboard_version}}/aio/deploy/recommended.yaml 62 | 63 | - name: Add Dashboard Ingress deployment descriptor 64 | template: 65 | src: dashboard-ingress.yaml.j2 66 | dest: "{{ install_temp_dir }}/dashboard-ingress.yaml" 67 | mode: 0644 68 | 69 | - name: Apply Dashboard Ingress 70 | shell: | 71 | kubectl apply -f {{ install_temp_dir }}/dashboard-ingress.yaml 72 | -------------------------------------------------------------------------------- /roles/kubernetes/tasks/kubernetes_common.yml: -------------------------------------------------------------------------------- 1 | # # # # # 2 | 3 | - name: Disable SELinux 4 | shell: | 5 | setenforce 0 6 | sed -i --follow-symlinks 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/sysconfig/selinux 7 | ignore_errors: yes 8 | 9 | #- name: Update firewall rules 10 | # shell: > 11 | # firewall-cmd --permanent --add-port=6443/tcp 12 | # firewall-cmd --permanent --add-port=2379-2380/tcp 13 | # firewall-cmd --permanent --add-port=10250/tcp 14 | # firewall-cmd --permanent --add-port=10251/tcp 15 | # firewall-cmd --permanent --add-port=10252/tcp 16 | # firewall-cmd --permanent --add-port=10255/tcp 17 | # firewall-cmd --complete-reload 18 | # modprobe br_netfilter 19 | # echo '1' > /proc/sys/net/bridge/bridge-nf-call-iptables 20 | 21 | - name: Remove swapfile from /etc/fstab 22 | mount: 23 | name: swap 24 | fstype: swap 25 | state: absent 26 | 27 | - name: Disable swap 28 | command: swapoff -a 29 | when: ansible_swaptotal_mb > 0 30 | 31 | - name: clean fyre proxy exports 32 | lineinfile: 33 | path: /etc/profile 34 | state: absent 35 | regexp: '^export.*(PROXY|proxy).*' 36 | 37 | - name: unset fyre proxy exports 38 | shell: | 39 | unset HTTP_PROXY 40 | unset FTP_PROXY 41 | unset NO_PROXY 42 | unset http_proxy 43 | unset ftp_proxy 44 | unset no_proxy 45 | 46 | - name: Add Kubernetes repository 47 | template: src="kubernetes.repo.j2" dest="/etc/yum.repos.d/kubernetes.repo" mode=0644 48 | 49 | #- name: Install Kubernetes components 50 | # shell: > 51 | # yum install -y docker kubelet kubeadm kubectl 52 | 53 | - name: Install Docker 54 | shell: > 55 | yum install -y docker 56 | 57 | - name: Initialize docker 58 | shell: > 59 | systemctl --force enable docker && systemctl start docker 60 | 61 | - name: Install Kubelet 62 | shell: > 63 | yum install -y kubelet-{{ kube_version }}.x86_64 64 | 65 | - name: Initialize kubelet 66 | shell: > 67 | systemctl --force enable kubelet && systemctl start kubelet 68 | 69 | - name: Install Kubeadm 70 | shell: > 71 | yum install -y kubeadm-{{ kube_version }}.x86_64 72 | 73 | - name: Install Kubectl 74 | shell: > 75 | yum install -y kubectl-{{ kube_version }}.x86_64 76 | 77 | - debug: 78 | msg: "Downloading crictl from: http://github.com/kubernetes-incubator/cri-tools/releases/download/{{ crictl_version }}/crictl-{{ crictl_version }}-linux-amd64.tar.gz" 79 | 80 | - name: Install crictl 81 | shell: | 82 | wget http://github.com/kubernetes-incubator/cri-tools/releases/download/{{ crictl_version }}/crictl-{{ crictl_version }}-linux-amd64.tar.gz 83 | tar -xvf crictl-{{ crictl_version }}-linux-amd64.tar.gz --directory /usr/bin/ 84 | rm -rf crictl-{{ crictl_version }}-linux-amd64.tar.gz 85 | 86 | - debug: 87 | msg: "Downloading critest from: http://github.com/kubernetes-incubator/cri-tools/releases/download/{{ crictl_version }}/critest-{{ crictl_version }}-linux-amd64.tar.gz" 88 | 89 | - name: Install critest 90 | shell: | 91 | wget http://github.com/kubernetes-incubator/cri-tools/releases/download/{{ crictl_version }}/critest-{{ crictl_version }}-linux-amd64.tar.gz 92 | tar -xvf critest-{{ crictl_version }}-linux-amd64.tar.gz --directory /usr/bin/ 93 | rm -rf critest-{{ crictl_version }}-linux-amd64.tar.gz 94 | 95 | - name: Set Kubernetes Config env variable 96 | template: 97 | src: kubernetes-env.sh.j2 98 | dest: /etc/profile.d/kubernetes-env.sh 99 | mode: 0644 100 | -------------------------------------------------------------------------------- /roles/kubernetes/tasks/kubernetes_master.yml: -------------------------------------------------------------------------------- 1 | - name: Reset the master 2 | shell: > 3 | kubeadm reset --force 4 | ignore_errors: yes 5 | 6 | - name: Remove old/obsolete configurations 7 | shell: > 8 | rm -rf /etc/kubernetes/ 9 | ignore_errors: yes 10 | 11 | - name: Bootstrap the master 12 | shell: | 13 | systemctl daemon-reload 14 | service kubelet restart 15 | kubeadm init --apiserver-advertise-address 0.0.0.0 --pod-network-cidr={{ kube_service_addresses }} --ignore-preflight-errors cri 16 | 17 | - name: Restart kubelet 18 | shell: | 19 | systemctl daemon-reload 20 | service kubelet restart 21 | 22 | # sleep 10; while true; do incomplete=$(kubectl get pod --all-namespaces | tail -n +2 | grep -Ev '([0-9]+)/\1') || break; sleep 10; done 23 | - name: Wait for all pods to be ready 24 | pause: 25 | minutes: 1 26 | 27 | - name: Print status of all pods 28 | shell: | 29 | kubectl get pod --all-namespaces 30 | 31 | - name: Create kubeadm token for joining nodes with 24h expiration (default) 32 | command: "kubeadm token create --print-join-command" 33 | register: temp_kubeadm_join_command 34 | 35 | - name: set kubeadm join command 36 | set_fact: kubeadm_join_command={{ temp_kubeadm_join_command.stdout }} 37 | 38 | -------------------------------------------------------------------------------- /roles/kubernetes/tasks/kubernetes_minion.yml: -------------------------------------------------------------------------------- 1 | 2 | - name: Reset the minion node 3 | shell: > 4 | kubeadm reset --force 5 | ignore_errors: yes 6 | 7 | - name: Join master node 8 | shell: | 9 | systemctl daemon-reload 10 | systemctl restart kubelet 11 | {{ hostvars.get(groups['master'][0]).get('kubeadm_join_command') }} --ignore-preflight-errors cri 12 | 13 | -------------------------------------------------------------------------------- /roles/kubernetes/tasks/main.yml: -------------------------------------------------------------------------------- 1 | - name: remove old install dir 2 | file: path= "{{ install_dir }}" state=absent 3 | 4 | - name: create install dir 5 | file: path="{{ install_dir }}" state=directory 6 | 7 | - name: remove old temporary install dir 8 | file: path="{{ install_temp_dir }}" state=absent 9 | 10 | - name: create temporary install dir 11 | file: path="{{ install_temp_dir }}" state=directory 12 | 13 | - import_tasks: kubernetes_common.yml 14 | 15 | - import_tasks: kubernetes_master.yml 16 | when: inventory_hostname in groups['master'] 17 | 18 | - import_tasks: kubernetes_minion.yml 19 | when: inventory_hostname not in groups['master'] 20 | 21 | - import_tasks: kubernetes_addons.yml 22 | when: inventory_hostname in groups['master'] 23 | -------------------------------------------------------------------------------- /roles/kubernetes/templates/dashboard-ingress.yaml.j2: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | kind: Ingress 3 | metadata: 4 | name: dashboard-ingress 5 | namespace: kubernetes-dashboard 6 | annotations: 7 | kubernetes.io/ingress.class: traefik 8 | traefik.frontend.rule.type: PathPrefixStrip 9 | spec: 10 | rules: 11 | - host: {{ groups['master'][0] }}.{{ ansible_domain }} 12 | http: 13 | paths: 14 | - path: /dashboard 15 | backend: 16 | serviceName: kubernetes-dashboard 17 | servicePort: 80 18 | # tls: 19 | # - secretName: traefik-tls-cert 20 | -------------------------------------------------------------------------------- /roles/kubernetes/templates/kubernetes-env.sh.j2: -------------------------------------------------------------------------------- 1 | export KUBECONFIG=/etc/kubernetes/admin.conf -------------------------------------------------------------------------------- /roles/kubernetes/templates/kubernetes.repo.j2: -------------------------------------------------------------------------------- 1 | [kubernetes] 2 | name=Kubernetes 3 | baseurl=https://packages.cloud.google.com/yum/repos/kubernetes-el7-x86_64 4 | enabled=1 5 | gpgcheck=1 6 | repo_gpgcheck=1 7 | gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg -------------------------------------------------------------------------------- /roles/kubernetes/templates/tiller-access.yaml.j2: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | name: tiller 5 | namespace: kube-system 6 | --- 7 | apiVersion: rbac.authorization.k8s.io/v1beta1 8 | kind: ClusterRoleBinding 9 | metadata: 10 | name: tiller 11 | roleRef: 12 | apiGroup: rbac.authorization.k8s.io 13 | kind: ClusterRole 14 | name: cluster-admin 15 | subjects: 16 | - kind: ServiceAccount 17 | name: tiller 18 | namespace: kube-system -------------------------------------------------------------------------------- /roles/kubernetes/templates/traefik-dashboard-ingress-route.yaml.j2: -------------------------------------------------------------------------------- 1 | apiVersion: traefik.containo.us/v1alpha1 2 | kind: IngressRoute 3 | metadata: 4 | name: traefik-dashboard 5 | spec: 6 | entryPoints: 7 | - web 8 | routes: 9 | - match: Host(`{{ groups['master'][0] }}.{{ ansible_domain }}`) && (PathPrefix(`/dashboard`) || PathPrefix(`/api`)) 10 | kind: Rule 11 | services: 12 | - name: api@internal 13 | kind: TraefikService 14 | -------------------------------------------------------------------------------- /roles/spark/defaults/main.yml: -------------------------------------------------------------------------------- 1 | spark: 2 | version: 2.2.0-k8s-0.5.0 3 | hadoop_version: 2.7.3 4 | # https://github.com/apache-spark-on-k8s/spark/releases/download/v2.2.0-kubernetes-0.5.0/spark-2.2.0-k8s-0.5.0-bin-with-hadoop-2.7.3.tgz 5 | download_location: https://github.com/apache-spark-on-k8s/spark/releases/download/v2.2.0-kubernetes-0.5.0 6 | -------------------------------------------------------------------------------- /roles/spark/tasks/main.yml: -------------------------------------------------------------------------------- 1 | - name: remove old install dir 2 | file: path= "{{ install_dir }}" state=absent 3 | 4 | - name: create install dir 5 | file: path="{{ install_dir }}" state=directory 6 | 7 | - name: remove old temporary install dir 8 | file: path="{{ install_temp_dir }}" state=absent 9 | 10 | - name: create temporary install dir 11 | file: path="{{ install_temp_dir }}" state=directory 12 | 13 | - import_tasks: spark.yml 14 | when: inventory_hostname in groups['master'] 15 | -------------------------------------------------------------------------------- /roles/spark/tasks/spark.yml: -------------------------------------------------------------------------------- 1 | 2 | - name: set spark installation path fact 3 | set_fact: spark_installation_dir=spark-{{ spark.version }}-bin-with-hadoop-{{ spark.hadoop_version }} 4 | 5 | - name: set spark archive fact 6 | set_fact: spark_archive=spark-{{ spark.version }}-bin-with-hadoop-{{ spark.hadoop_version }}.tgz 7 | 8 | - name: set spark download location fact 9 | set_fact: spark_download={{ spark.download_location }}/{{ spark_archive }} 10 | 11 | - debug: 12 | msg: "Downloading Spark from: {{ spark_download }}" 13 | 14 | - name: download spark 15 | get_url: url="{{ spark_download }}" dest="{{ install_temp_dir }}/{{ spark_archive }}" 16 | 17 | - name: create install directory 18 | file: 19 | path: "{{ install_dir }}/{{ spark_installation_dir }}" 20 | state: directory 21 | 22 | - name: unarchive to the install directory 23 | shell: "tar --warning=no-unknown-keyword -zxf {{ install_temp_dir }}/{{ spark_archive }} --strip 1 --directory {{ install_dir }}/{{ spark_installation_dir }}" 24 | 25 | # Environment setup. 26 | - name: add spark profile to startup 27 | template: 28 | src: spark-env.sh.j2 29 | dest: /etc/profile.d/spark-env.sh 30 | mode: 0644 31 | -------------------------------------------------------------------------------- /roles/spark/templates/spark-env.sh.j2: -------------------------------------------------------------------------------- 1 | export SPARK_HOME={{ install_dir }}/{{ spark_installation_dir }} 2 | 3 | export PATH=$SPARK_HOME/bin:$SPARK_HOME/sbin:$PATH 4 | 5 | alias cds="cd %SPARK_HOME" 6 | 7 | -------------------------------------------------------------------------------- /setup-kubernetes.yml: -------------------------------------------------------------------------------- 1 | - name: setup kubernetes 2 | hosts: all 3 | remote_user: root 4 | roles: 5 | - role: common 6 | - role: kubernetes 7 | - role: jupyter-enterprise-gateway 8 | - role: jupyterhub 9 | -------------------------------------------------------------------------------- /setup-spark.yml: -------------------------------------------------------------------------------- 1 | - name: setup spark 2 | hosts: all 3 | remote_user: root 4 | roles: 5 | - role: common 6 | - role: spark 7 | --------------------------------------------------------------------------------