├── ansible.cfg ├── current_cluster └── .keep ├── hosts ├── images └── .keep ├── readme.md ├── roles ├── configure-nodes │ └── tasks │ │ └── main.yml ├── create-k8s-master │ └── tasks │ │ └── main.yml ├── install-weave │ └── tasks │ │ └── main.yml ├── join-workers │ └── tasks │ │ └── main.yml └── wrapup │ ├── files │ ├── xxx.yml │ └── yyy.yml │ └── tasks │ └── main.yml ├── top.retry └── top.yml /ansible.cfg: -------------------------------------------------------------------------------- 1 | # config file for ansible -- https://ansible.com/ 2 | # =============================================== 3 | 4 | # nearly all parameters can be overridden in ansible-playbook 5 | # or with command line flags. ansible will read ANSIBLE_CONFIG, 6 | # ansible.cfg in the current working directory, .ansible.cfg in 7 | # the home directory or /etc/ansible/ansible.cfg, whichever it 8 | # finds first 9 | 10 | [defaults] 11 | 12 | # some basic default values... 13 | 14 | inventory = hosts 15 | #library = /usr/share/my_modules/ 16 | #module_utils = /usr/share/my_module_utils/ 17 | #remote_tmp = ~/.ansible/tmp 18 | #local_tmp = ~/.ansible/tmp 19 | #plugin_filters_cfg = /etc/ansible/plugin_filters.yml 20 | #forks = 5 21 | #poll_interval = 15 22 | #sudo_user = root 23 | #ask_sudo_pass = True 24 | #ask_pass = True 25 | #transport = smart 26 | #remote_port = 22 27 | #module_lang = C 28 | #module_set_locale = False 29 | 30 | # plays will gather facts by default, which contain information about 31 | # the remote system. 32 | # 33 | # smart - gather by default, but don't regather if already gathered 34 | # implicit - gather by default, turn off with gather_facts: False 35 | # explicit - do not gather by default, must say gather_facts: True 36 | #gathering = implicit 37 | 38 | # This only affects the gathering done by a play's gather_facts directive, 39 | # by default gathering retrieves all facts subsets 40 | # all - gather all subsets 41 | # network - gather min and network facts 42 | # hardware - gather hardware facts (longest facts to retrieve) 43 | # virtual - gather min and virtual facts 44 | # facter - import facts from facter 45 | # ohai - import facts from ohai 46 | # You can combine them using comma (ex: network,virtual) 47 | # You can negate them using ! (ex: !hardware,!facter,!ohai) 48 | # A minimal set of facts is always gathered. 49 | #gather_subset = all 50 | 51 | # some hardware related facts are collected 52 | # with a maximum timeout of 10 seconds. This 53 | # option lets you increase or decrease that 54 | # timeout to something more suitable for the 55 | # environment. 56 | # gather_timeout = 10 57 | 58 | # additional paths to search for roles in, colon separated 59 | #roles_path = /etc/ansible/roles 60 | 61 | # uncomment this to disable SSH key host checking 62 | #host_key_checking = False 63 | 64 | # change the default callback, you can only have one 'stdout' type enabled at a time. 65 | #stdout_callback = skippy 66 | 67 | 68 | ## Ansible ships with some plugins that require whitelisting, 69 | ## this is done to avoid running all of a type by default. 70 | ## These setting lists those that you want enabled for your system. 71 | ## Custom plugins should not need this unless plugin author specifies it. 72 | 73 | # enable callback plugins, they can output to stdout but cannot be 'stdout' type. 74 | #callback_whitelist = timer, mail 75 | 76 | # Determine whether includes in tasks and handlers are "static" by 77 | # default. As of 2.0, includes are dynamic by default. Setting these 78 | # values to True will make includes behave more like they did in the 79 | # 1.x versions. 80 | #task_includes_static = False 81 | #handler_includes_static = False 82 | 83 | # Controls if a missing handler for a notification event is an error or a warning 84 | #error_on_missing_handler = True 85 | 86 | # change this for alternative sudo implementations 87 | #sudo_exe = sudo 88 | 89 | # What flags to pass to sudo 90 | # WARNING: leaving out the defaults might create unexpected behaviours 91 | #sudo_flags = -H -S -n 92 | 93 | # SSH timeout 94 | #timeout = 10 95 | 96 | # default user to use for playbooks if user is not specified 97 | # (/usr/bin/ansible will use current user as default) 98 | #remote_user = root 99 | 100 | # logging is off by default unless this path is defined 101 | # if so defined, consider logrotate 102 | #log_path = /var/log/ansible.log 103 | 104 | # default module name for /usr/bin/ansible 105 | #module_name = command 106 | 107 | # use this shell for commands executed under sudo 108 | # you may need to change this to bin/bash in rare instances 109 | # if sudo is constrained 110 | #executable = /bin/sh 111 | 112 | # if inventory variables overlap, does the higher precedence one win 113 | # or are hash values merged together? The default is 'replace' but 114 | # this can also be set to 'merge'. 115 | #hash_behaviour = replace 116 | 117 | # by default, variables from roles will be visible in the global variable 118 | # scope. To prevent this, the following option can be enabled, and only 119 | # tasks and handlers within the role will see the variables there 120 | #private_role_vars = yes 121 | 122 | # list any Jinja2 extensions to enable here: 123 | #jinja2_extensions = jinja2.ext.do,jinja2.ext.i18n 124 | 125 | # if set, always use this private key file for authentication, same as 126 | # if passing --private-key to ansible or ansible-playbook 127 | #private_key_file = /path/to/file 128 | 129 | # If set, configures the path to the Vault password file as an alternative to 130 | # specifying --vault-password-file on the command line. 131 | #vault_password_file = /path/to/vault_password_file 132 | 133 | # format of string {{ ansible_managed }} available within Jinja2 134 | # templates indicates to users editing templates files will be replaced. 135 | # replacing {file}, {host} and {uid} and strftime codes with proper values. 136 | #ansible_managed = Ansible managed: {file} modified on %Y-%m-%d %H:%M:%S by {uid} on {host} 137 | # {file}, {host}, {uid}, and the timestamp can all interfere with idempotence 138 | # in some situations so the default is a static string: 139 | #ansible_managed = Ansible managed 140 | 141 | # by default, ansible-playbook will display "Skipping [host]" if it determines a task 142 | # should not be run on a host. Set this to "False" if you don't want to see these "Skipping" 143 | # messages. NOTE: the task header will still be shown regardless of whether or not the 144 | # task is skipped. 145 | #display_skipped_hosts = True 146 | 147 | # by default, if a task in a playbook does not include a name: field then 148 | # ansible-playbook will construct a header that includes the task's action but 149 | # not the task's args. This is a security feature because ansible cannot know 150 | # if the *module* considers an argument to be no_log at the time that the 151 | # header is printed. If your environment doesn't have a problem securing 152 | # stdout from ansible-playbook (or you have manually specified no_log in your 153 | # playbook on all of the tasks where you have secret information) then you can 154 | # safely set this to True to get more informative messages. 155 | #display_args_to_stdout = False 156 | 157 | # by default (as of 1.3), Ansible will raise errors when attempting to dereference 158 | # Jinja2 variables that are not set in templates or action lines. Uncomment this line 159 | # to revert the behavior to pre-1.3. 160 | #error_on_undefined_vars = False 161 | 162 | # by default (as of 1.6), Ansible may display warnings based on the configuration of the 163 | # system running ansible itself. This may include warnings about 3rd party packages or 164 | # other conditions that should be resolved if possible. 165 | # to disable these warnings, set the following value to False: 166 | #system_warnings = True 167 | 168 | # by default (as of 1.4), Ansible may display deprecation warnings for language 169 | # features that should no longer be used and will be removed in future versions. 170 | # to disable these warnings, set the following value to False: 171 | #deprecation_warnings = True 172 | 173 | # (as of 1.8), Ansible can optionally warn when usage of the shell and 174 | # command module appear to be simplified by using a default Ansible module 175 | # instead. These warnings can be silenced by adjusting the following 176 | # setting or adding warn=yes or warn=no to the end of the command line 177 | # parameter string. This will for example suggest using the git module 178 | # instead of shelling out to the git command. 179 | # command_warnings = False 180 | 181 | 182 | # set plugin path directories here, separate with colons 183 | #action_plugins = /usr/share/ansible/plugins/action 184 | #cache_plugins = /usr/share/ansible/plugins/cache 185 | #callback_plugins = /usr/share/ansible/plugins/callback 186 | #connection_plugins = /usr/share/ansible/plugins/connection 187 | #lookup_plugins = /usr/share/ansible/plugins/lookup 188 | #inventory_plugins = /usr/share/ansible/plugins/inventory 189 | #vars_plugins = /usr/share/ansible/plugins/vars 190 | #filter_plugins = /usr/share/ansible/plugins/filter 191 | #test_plugins = /usr/share/ansible/plugins/test 192 | #terminal_plugins = /usr/share/ansible/plugins/terminal 193 | #strategy_plugins = /usr/share/ansible/plugins/strategy 194 | 195 | 196 | # by default, ansible will use the 'linear' strategy but you may want to try 197 | # another one 198 | #strategy = free 199 | 200 | # by default callbacks are not loaded for /bin/ansible, enable this if you 201 | # want, for example, a notification or logging callback to also apply to 202 | # /bin/ansible runs 203 | #bin_ansible_callbacks = False 204 | 205 | 206 | # don't like cows? that's unfortunate. 207 | # set to 1 if you don't want cowsay support or export ANSIBLE_NOCOWS=1 208 | #nocows = 1 209 | 210 | # set which cowsay stencil you'd like to use by default. When set to 'random', 211 | # a random stencil will be selected for each task. The selection will be filtered 212 | # against the `cow_whitelist` option below. 213 | #cow_selection = default 214 | #cow_selection = random 215 | 216 | # when using the 'random' option for cowsay, stencils will be restricted to this list. 217 | # it should be formatted as a comma-separated list with no spaces between names. 218 | # NOTE: line continuations here are for formatting purposes only, as the INI parser 219 | # in python does not support them. 220 | #cow_whitelist=bud-frogs,bunny,cheese,daemon,default,dragon,elephant-in-snake,elephant,eyes,\ 221 | # hellokitty,kitty,luke-koala,meow,milk,moofasa,moose,ren,sheep,small,stegosaurus,\ 222 | # stimpy,supermilker,three-eyes,turkey,turtle,tux,udder,vader-koala,vader,www 223 | 224 | # don't like colors either? 225 | # set to 1 if you don't want colors, or export ANSIBLE_NOCOLOR=1 226 | #nocolor = 1 227 | 228 | # if set to a persistent type (not 'memory', for example 'redis') fact values 229 | # from previous runs in Ansible will be stored. This may be useful when 230 | # wanting to use, for example, IP information from one group of servers 231 | # without having to talk to them in the same playbook run to get their 232 | # current IP information. 233 | #fact_caching = memory 234 | 235 | 236 | # retry files 237 | # When a playbook fails by default a .retry file will be created in ~/ 238 | # You can disable this feature by setting retry_files_enabled to False 239 | # and you can change the location of the files by setting retry_files_save_path 240 | 241 | #retry_files_enabled = False 242 | #retry_files_save_path = ~/.ansible-retry 243 | 244 | # squash actions 245 | # Ansible can optimise actions that call modules with list parameters 246 | # when looping. Instead of calling the module once per with_ item, the 247 | # module is called once with all items at once. Currently this only works 248 | # under limited circumstances, and only with parameters named 'name'. 249 | #squash_actions = apk,apt,dnf,homebrew,pacman,pkgng,yum,zypper 250 | 251 | # prevents logging of task data, off by default 252 | #no_log = False 253 | 254 | # prevents logging of tasks, but only on the targets, data is still logged on the master/controller 255 | #no_target_syslog = False 256 | 257 | # controls whether Ansible will raise an error or warning if a task has no 258 | # choice but to create world readable temporary files to execute a module on 259 | # the remote machine. This option is False by default for security. Users may 260 | # turn this on to have behaviour more like Ansible prior to 2.1.x. See 261 | # https://docs.ansible.com/ansible/become.html#becoming-an-unprivileged-user 262 | # for more secure ways to fix this than enabling this option. 263 | #allow_world_readable_tmpfiles = False 264 | 265 | # controls the compression level of variables sent to 266 | # worker processes. At the default of 0, no compression 267 | # is used. This value must be an integer from 0 to 9. 268 | #var_compression_level = 9 269 | 270 | # controls what compression method is used for new-style ansible modules when 271 | # they are sent to the remote system. The compression types depend on having 272 | # support compiled into both the controller's python and the client's python. 273 | # The names should match with the python Zipfile compression types: 274 | # * ZIP_STORED (no compression. available everywhere) 275 | # * ZIP_DEFLATED (uses zlib, the default) 276 | # These values may be set per host via the ansible_module_compression inventory 277 | # variable 278 | #module_compression = 'ZIP_DEFLATED' 279 | 280 | # This controls the cutoff point (in bytes) on --diff for files 281 | # set to 0 for unlimited (RAM may suffer!). 282 | #max_diff_size = 1048576 283 | 284 | # This controls how ansible handles multiple --tags and --skip-tags arguments 285 | # on the CLI. If this is True then multiple arguments are merged together. If 286 | # it is False, then the last specified argument is used and the others are ignored. 287 | # This option will be removed in 2.8. 288 | #merge_multiple_cli_flags = True 289 | 290 | # Controls showing custom stats at the end, off by default 291 | #show_custom_stats = True 292 | 293 | # Controls which files to ignore when using a directory as inventory with 294 | # possibly multiple sources (both static and dynamic) 295 | #inventory_ignore_extensions = ~, .orig, .bak, .ini, .cfg, .retry, .pyc, .pyo 296 | 297 | # This family of modules use an alternative execution path optimized for network appliances 298 | # only update this setting if you know how this works, otherwise it can break module execution 299 | #network_group_modules=eos, nxos, ios, iosxr, junos, vyos 300 | 301 | # When enabled, this option allows lookups (via variables like {{lookup('foo')}} or when used as 302 | # a loop with `with_foo`) to return data that is not marked "unsafe". This means the data may contain 303 | # jinja2 templating language which will be run through the templating engine. 304 | # ENABLING THIS COULD BE A SECURITY RISK 305 | #allow_unsafe_lookups = False 306 | 307 | # set default errors for all plays 308 | #any_errors_fatal = False 309 | 310 | [inventory] 311 | # enable inventory plugins, default: 'host_list', 'script', 'yaml', 'ini' 312 | #enable_plugins = host_list, virtualbox, yaml, constructed 313 | 314 | # ignore these extensions when parsing a directory as inventory source 315 | #ignore_extensions = .pyc, .pyo, .swp, .bak, ~, .rpm, .md, .txt, ~, .orig, .ini, .cfg, .retry 316 | 317 | # ignore files matching these patterns when parsing a directory as inventory source 318 | #ignore_patterns= 319 | 320 | # If 'true' unparsed inventory sources become fatal errors, they are warnings otherwise. 321 | #unparsed_is_failed=False 322 | 323 | [privilege_escalation] 324 | #become=True 325 | #become_method=sudo 326 | #become_user=root 327 | #become_ask_pass=False 328 | 329 | [paramiko_connection] 330 | 331 | # uncomment this line to cause the paramiko connection plugin to not record new host 332 | # keys encountered. Increases performance on new host additions. Setting works independently of the 333 | # host key checking setting above. 334 | #record_host_keys=False 335 | 336 | # by default, Ansible requests a pseudo-terminal for commands executed under sudo. Uncomment this 337 | # line to disable this behaviour. 338 | #pty=False 339 | 340 | # paramiko will default to looking for SSH keys initially when trying to 341 | # authenticate to remote devices. This is a problem for some network devices 342 | # that close the connection after a key failure. Uncomment this line to 343 | # disable the Paramiko look for keys function 344 | #look_for_keys = False 345 | 346 | # When using persistent connections with Paramiko, the connection runs in a 347 | # background process. If the host doesn't already have a valid SSH key, by 348 | # default Ansible will prompt to add the host key. This will cause connections 349 | # running in background processes to fail. Uncomment this line to have 350 | # Paramiko automatically add host keys. 351 | #host_key_auto_add = True 352 | 353 | [ssh_connection] 354 | 355 | # ssh arguments to use 356 | # Leaving off ControlPersist will result in poor performance, so use 357 | # paramiko on older platforms rather than removing it, -C controls compression use 358 | #ssh_args = -C -o ControlMaster=auto -o ControlPersist=60s 359 | 360 | # The base directory for the ControlPath sockets. 361 | # This is the "%(directory)s" in the control_path option 362 | # 363 | # Example: 364 | # control_path_dir = /tmp/.ansible/cp 365 | #control_path_dir = ~/.ansible/cp 366 | 367 | # The path to use for the ControlPath sockets. This defaults to a hashed string of the hostname, 368 | # port and username (empty string in the config). The hash mitigates a common problem users 369 | # found with long hostames and the conventional %(directory)s/ansible-ssh-%%h-%%p-%%r format. 370 | # In those cases, a "too long for Unix domain socket" ssh error would occur. 371 | # 372 | # Example: 373 | # control_path = %(directory)s/%%h-%%r 374 | #control_path = 375 | 376 | # Enabling pipelining reduces the number of SSH operations required to 377 | # execute a module on the remote server. This can result in a significant 378 | # performance improvement when enabled, however when using "sudo:" you must 379 | # first disable 'requiretty' in /etc/sudoers 380 | # 381 | # By default, this option is disabled to preserve compatibility with 382 | # sudoers configurations that have requiretty (the default on many distros). 383 | # 384 | #pipelining = False 385 | 386 | # Control the mechanism for transferring files (old) 387 | # * smart = try sftp and then try scp [default] 388 | # * True = use scp only 389 | # * False = use sftp only 390 | #scp_if_ssh = smart 391 | 392 | # Control the mechanism for transferring files (new) 393 | # If set, this will override the scp_if_ssh option 394 | # * sftp = use sftp to transfer files 395 | # * scp = use scp to transfer files 396 | # * piped = use 'dd' over SSH to transfer files 397 | # * smart = try sftp, scp, and piped, in that order [default] 398 | #transfer_method = smart 399 | 400 | # if False, sftp will not use batch mode to transfer files. This may cause some 401 | # types of file transfer failures impossible to catch however, and should 402 | # only be disabled if your sftp version has problems with batch mode 403 | #sftp_batch_mode = False 404 | 405 | # The -tt argument is passed to ssh when pipelining is not enabled because sudo 406 | # requires a tty by default. 407 | #use_tty = True 408 | 409 | [persistent_connection] 410 | 411 | # Configures the persistent connection timeout value in seconds. This value is 412 | # how long the persistent connection will remain idle before it is destroyed. 413 | # If the connection doesn't receive a request before the timeout value 414 | # expires, the connection is shutdown. The default value is 30 seconds. 415 | #connect_timeout = 30 416 | 417 | # Configures the persistent connection retry timeout. This value configures the 418 | # the retry timeout that ansible-connection will wait to connect 419 | # to the local domain socket. This value must be larger than the 420 | # ssh timeout (timeout) and less than persistent connection idle timeout (connect_timeout). 421 | # The default value is 15 seconds. 422 | #connect_retry_timeout = 15 423 | 424 | # The command timeout value defines the amount of time to wait for a command 425 | # or RPC call before timing out. The value for the command timeout must 426 | # be less than the value of the persistent connection idle timeout (connect_timeout) 427 | # The default value is 10 second. 428 | #command_timeout = 10 429 | 430 | [accelerate] 431 | #accelerate_port = 5099 432 | #accelerate_timeout = 30 433 | #accelerate_connect_timeout = 5.0 434 | 435 | # The daemon timeout is measured in minutes. This time is measured 436 | # from the last activity to the accelerate daemon. 437 | #accelerate_daemon_timeout = 30 438 | 439 | # If set to yes, accelerate_multi_key will allow multiple 440 | # private keys to be uploaded to it, though each user must 441 | # have access to the system via SSH to add a new key. The default 442 | # is "no". 443 | #accelerate_multi_key = yes 444 | 445 | [selinux] 446 | # file systems that require special treatment when dealing with security context 447 | # the default behaviour that copies the existing context or uses the user default 448 | # needs to be changed to use the file system dependent context. 449 | #special_context_filesystems=nfs,vboxsf,fuse,ramfs,9p 450 | 451 | # Set this to yes to allow libvirt_lxc connections to work without SELinux. 452 | #libvirt_lxc_noseclabel = yes 453 | 454 | [colors] 455 | #highlight = white 456 | #verbose = blue 457 | #warn = bright purple 458 | #error = red 459 | #debug = dark gray 460 | #deprecate = purple 461 | #skip = cyan 462 | #unreachable = red 463 | #ok = green 464 | #changed = yellow 465 | #diff_add = green 466 | #diff_remove = red 467 | #diff_lines = cyan 468 | 469 | 470 | [diff] 471 | # Always print diff when running ( same as always running with -D/--diff ) 472 | # always = no 473 | 474 | # Set how many context lines to show in diff 475 | # context = 3 476 | -------------------------------------------------------------------------------- /current_cluster/.keep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opsview/kubernetes-ansible-example/6b7ed923a737360cd17ac355cbdf4d79cc6a8c6b/current_cluster/.keep -------------------------------------------------------------------------------- /hosts: -------------------------------------------------------------------------------- 1 | # This is the default ansible 'hosts' file. 2 | # 3 | # It should live in /etc/ansible/hosts 4 | # 5 | # - Comments begin with the '#' character 6 | # - Blank lines are ignored 7 | # - Groups of hosts are delimited by [header] elements 8 | # - You can enter hostnames or ip addresses 9 | # - A hostname/ip can be a member of multiple groups 10 | 11 | # Ex 1: Ungrouped hosts, specify before any group headers 12 | 13 | k8smaster 14 | k8sworker1 15 | k8sworker2 16 | -------------------------------------------------------------------------------- /images/.keep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opsview/kubernetes-ansible-example/6b7ed923a737360cd17ac355cbdf4d79cc6a8c6b/images/.keep -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | # kubernetes-ansible-example 2 | ## Create a Kubernetes dev/demo cluster on VMs, using Ansible 3 | 4 | **This repo contains Ansible roles that let you quickly create a multi-node (master node plus worker nodes) Kubernetes cluster on virtual (or, with small modifications, on bare metal) machines. The current version has been tested on VirtualBox for Windows10 and Ubuntu Linux. Future revisions will be tested on Amazon EC2, OpenStack, Azure, GCE, on-premises bare metal, hosted bare metal, and other platforms.** 5 | 6 | ### Use at Own Risk 7 | _This repo is not for production use in any context, and is not supported by Opsview Product, Engineering, or Customer Success teams._ It is maintained by Opsview's Marketing and Innovation teams to support tutorials, videos, presentations, webinars, and other content published on Opsview.com on topics such as: 8 | 9 | - Kubernetes, Docker, Linux, database, and hardware monitoring with Opsview Monitor 10 | - Operating and monitoring on-premises serverless computing (with OpenFaaS and other platforms) 11 | - Future tutorials on AWS, Azure and other kinds of cloud monitoring. 12 | 13 | ### Contents 14 | [Kubernetes Cluster Description](#k8s-cluster-description) 15 | [Prerequisites](#prerequisites) 16 | [Getting Started](#getting-started) 17 | [Configuring Nodes](#configuring-nodes) 18 |     [Node Hostnames and Admin Usernames](#node-hostnames-and-admin-usernames) 19 |     [Modifying /etc/hosts](#modifying-etc-hosts) 20 |     [Setting up Passwordless Access](#setting-up-passwordless-access) 21 | [Deploying Kubernetes](#deploying-k8s) 22 | [Deployment Phases](#deployment-phases) 23 | [Engage kubectl proxy](#engage-proxy) 24 | [Dashboard Access](#dashboard-access) 25 |     [Authenticating to the Dashboard](#auth-to-dash) 26 | [Flight Checks](#flight-checks) 27 | [Resources](#resources) 28 | [Opsview Tutorials](#opsview-tutorials) 29 | 30 | ### Kubernetes Cluster description 31 | These Ansible roles and playbooks create a simple Kubernetes cluster on virtual machines preconfigured with Ubuntu 16.04 server as base OS. The deployment is automated using the Kubernetes Project's _kubeadm_ toolkit. The resulting cluster is identical to what is described in the article entitled _[Using kubeadm to Create a Cluster](https://kubernetes.io/docs/setup/independent/create-cluster-kubeadm/)_ in official Kubernetes documentation. 32 | 33 | The cluster includes: 34 | 35 | - One Kubernetes master node plus N worker nodes (default = 2 workers) 36 | - Weave.net CNI networking 37 | - kubectl CLI 38 | - Kubernetes Dashboard 39 | 40 | The playbooks also install: 41 | 42 | - Opsview Monitor agent (and dependencies) 43 | This allows easy setup of full stack monitoring with Opsview Monitor (deployed separately) 44 | 45 | The playbooks "de-taint" the master node to allow container workloads to be deployed there. This is not production best-practice, but provides more hosting capacity in small-scale implementations. 46 | 47 | The deployment roles and playbooks can be customized to: 48 | 49 | - Create clusters with more or fewer worker nodes (easy) 50 | - Use alternative CNI networking fabrics (e.g., Calico, Flannel, etc.) (easy) 51 | - _Not_ de-taint the master node (easy) 52 | - Install additional Kubernetes components and tools (mileage varies) 53 | - Use a different Ubuntu version or Linux distro as base OS (mileage varies) 54 | 55 | ### Prerequisites 56 | 1. For Kubernetes nodes: Three or more virtual (or bare-metal) machines running [Ubuntu 16.04 LTS server](http://releases.ubuntu.com/16.04/ubuntu-16.04.4-server-amd64.iso.torrent?_ga=2.138459893.771820039.1526448425-1018235263.1526448425). Ubuntu 18.04 LTS will probably work as well, but this has not yet been tested. 57 | 1. For deployment and management: One VM or bare-metal machine running [Ansible 2.5+](http://docs.ansible.com/ansible/latest/installation_guide/intro_installation.html#installation-guide) on Ubuntu Desktop or other host OS. This machine should also be set up with standard utilities for terminal, ssh key management, text editing (e.g., [Atom](https://atom.io/)), [Git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git), and a standard browser such as Firefox or Chrome. 58 | 1. 2 GB or more of RAM per machine (recommended: as much as possible) 59 | 1. 2 CPUs or more on the master (recommended: as many vCPUs as possible) 60 | 1. At least 20GB (preferably more) of hard disk or (preferably) SSD space 61 | 1. Full network connectivity between all machines (public or private network is fine). If using VirtualBox, a single vNIC per machine with 'bridged' or NAT' networking should enable machines to see one another and access internet. 62 | 63 | ### Getting Started 64 | Install Ansible, then begin by cloning this repo to your home directory on your management machine: 65 | 66 | ``` 67 | cd /home/(your home directory name) 68 | git clone https://github.com/opsview/kubernetes-ansible-example.git 69 | ``` 70 | 71 | This will create a folder called **kubernetes-ansible-example** from the top level of which you can execute the playbooks directly. 72 | 73 | If you're using Atom or similar, project-based text-editor, open it up and add the kubernetes-ansible-example repo as a project folder, letting you review the entire contents of the folder tree easily. 74 | 75 | ### Configuring Nodes 76 | The Ansible roles and playbooks assume that you are configuring a total of three VMs for use as Kubernetes nodes (one master and two workers) with hostnames _k8smaster_, _k8sworker1_, and _k8sworker2_. These names are arbitrary -- if you use different names, use Atom's multi-file search and replace to replace our default names with your own. Pay particular attention to getting the names of the hosts correct in the file **kubernetes-ansible-example/hosts**. 77 | 78 | Install Ubuntu 16.04 LTS on each of the VMs you'll be using as nodes. This [tutorial](https://medium.com/@tushar0618/install-ubuntu-16-04-lts-on-virtual-box-desktop-version-30dc6f1958d0) shows you how to do this on VirtualBox. 79 | 80 | #### Node Hostnames and Admin Usernames 81 | During the Ubuntu installation process for each node machine, you will be asked to provide a hostname. If you intend to keep our default hostnames, provide the names _k8smaster_, _k8sworker1_, and _k8sworker2_. 82 | 83 | You will also be asked to name an administrator user (automatically added to _sudoers_) and provide a password. We recommend using the name _k8suser_, but this is arbitrary. We will be setting up key-based/passwordless access on these machines for convenience in using Ansible. 84 | 85 | Set up the disk as you prefer -- defaults (i.e., guided install) are fine. Note that the playbooks will turn swap off on nodes, which is required by Kubernetes. 86 | 87 | Once Ubuntu is installed, reboot the nodes. 88 | 89 | #### Modifying /etc/hosts 90 | To facilitate terminal access to your nodes and simplify use of Ansible (but avoid the need to set up a DNS), we recommend adding your node hosts to your management machine's /etc/hosts file, per the following example. This can be done using: 91 | 92 | ``` 93 | sudo atom /etc/hosts 94 | ``` 95 | Example: 96 | 97 | ``` 98 | 127.0.0.1 localhost 99 | 127.0.1.1 management-machine-hostname 100 | 101 | k8smaster 102 | k8sworker1 103 | k8sworker2 104 | 105 | # ... etc. 106 | 107 | ``` 108 | Once changes are made, save the /etc/hosts file, then logout and back into your management machine (or restart it). 109 | 110 | Test that you can ssh into your node machines with your password: 111 | 112 | ``` 113 | user@management-machine-hostname:~$ ssh k8suser@k8smaster 114 | k8suser@k8smaster's password: 115 | Welcome to Ubuntu 16.04.4 LTS (GNU/Linux 4.4.0-116-generic x86_64) 116 | 117 | * Documentation: https://help.ubuntu.com 118 | * Management: https://landscape.canonical.com 119 | * Support: https://ubuntu.com/advantage 120 | 121 | 68 packages can be updated. 122 | 25 updates are security updates. 123 | 124 | 125 | Last login: Wed May 16 02:29:26 2018 126 | To run a command as administrator (user "root"), use "sudo ". 127 | See "man sudo_root" for details. 128 | 129 | k8suser@k8smaster:~$ 130 | ``` 131 | #### Setting Up Passwordless Access 132 | To speed access to your nodes via ssh and to make life easier for Ansible, you should set up passwordless access to your Kubernetes node machines as follows: 133 | 134 | ###### Create an SSH Key 135 | Create an ssh key, following the guidance in [this tutorial](https://www.digitalocean.com/community/tutorials/how-to-set-up-ssh-keys-on-ubuntu-1604) and save it in the default location (/your_home/.ssh). Do not protect your key with a password. 136 | 137 | ###### Copy your Key to the Nodes 138 | Using the same tutorial as guide, copy your key to each node, providing your password as requested: 139 | 140 | Example: 141 | ``` 142 | ssh-copy-id k8suser@k8smaster 143 | ``` 144 | Copy the key to each Kubernetes node machine in the same way. 145 | 146 | ###### Test Passwordless Login 147 | You should now test to see that you can log in to your machines as follows, without a password: 148 | 149 | Example: 150 | 151 | ``` 152 | ssh k8suser@k8smaster 153 | ``` 154 | 155 | ### Deploying Kubernetes 156 | You can now initiate Ansible deployment of your Kubernetes cluster from the toplevel of the folder **kubernetes-ansible-example** as follows: 157 | 158 | ``` 159 | cd kubernetes-ansible-example 160 | ansible-playbook -u k8suser -K top.yml 161 | ``` 162 | ... executing the toplevel playbook file, 'top.yml,' as user k8suser. 163 | 164 | ### Deployment Phases 165 | 166 | The top.yml playbook begins by installing Python 2 on all nodes (not, strictly speaking, an Ansible requirement -- Ansible can use Python 3, which is installed by default on Ubuntu -- but part of "standard" Ansible deployment patterns). Thereafter, deployment is carried out in several phases, by roles. To parse what's going on, it can be helpful to read the article _[Using kubeadm to Create a Cluster](https://kubernetes.io/docs/setup/independent/create-cluster-kubeadm/)_ carefully: 167 | 168 | 1. **The configure-nodes role is executed on all three nodes** (see **kubernetes-ansible-example/configure-nodes/tasks/main.yml**) 169 | - The behavior of the vi editor is corrected for later convenience. 170 | - Swap is turned off and the change made permanent. 171 | - The server is restarted. 172 | - Opsview agent dependencies are installed. 173 | - The Opsview agent is installed. 174 | - Docker (a Kubernetes dependency) is installed. 175 | - The docker.service file is updated and the daemon restarted. 176 | - A Perl dependency for Opsview Docker monitoring is installed. 177 | - apt-transport-https is installed to enable installation of deb packages from remote web URLs. 178 | - Google GPG keys and repos are added and enabled. 179 | - The kubelet service is installed and started. 180 | - kubeadm (the deployer) and kubectl (the Kubernetes CLI) are installed. 181 | - The Kubernetes cgroup driver is matched to the one in use by Docker. 182 | - kubelet is restarted on each machine, and waits in a 'crashloop' (normal) 183 | 184 | 1. **The create-k8s-master role is executed on the k8smaster node** (see **kubernetes-ansible-example/create-k8s-master/tasks/main.yml**) 185 | This involves executing **kubeadm init** on the master, creating the master node, then recovering kubeadm's output to preserve the returned **kubeadm join** command that will later be used to join workers to the master node. The returned output is saved as .txt in the **kubernetes-ansible-example/current-cluster** directory. Note: the Weave CNI network that we install in step 3, below, does not require special parameterization of **kubeadm init** -- however, if you decide to change network fabrics, most of the other CNI network options do require parameters to be inserted here. 186 | 187 | 1. **The install-weave role is executed on the k8smaster node** (see **kubernetes-ansible-example/install-weave/tasks/main.yml**) 188 | This involves making an iptables change required for Weave.Net, then setting up a new user (kubeuser) as a lower-permission user of kubectl -- doing this here so that associated .conf files can then be exported for use by the weave installer. Finally, the installer is applied, starting weave containers on the Kubernetes master node. 189 | 190 | 1. **The wrapup role is executed on the k8smaster node** (see **kubernetes-ansible-example/wrapup/tasks/main.yml**) 191 | This role de-taints the master node to permit general workloads to run there. It then installs the Kubernetes Dashboard, creates a user context for authenticating to it, runs **kubectl kube-system describe secrets** to dump cluster secrets and parses out a so-called 'bearer token' that will serve as a password for logging into the Dashboard, saving this on the management machine's local file system (in **kubernetes-ansible-example/current-cluster**). Finally, it makes the non-administrative Kubernetes user able to call Docker without sudoing. 192 | 193 | 1. **The join-workers role is executed on the k8sworker1 and k8sworker2 nodes** (see **kubernetes-ansible-example/join-workers/tasks/main.yml**) 194 | Here, the **kubeadm join** command returned by **kubeadm init** is extracted from returned output and executed on each of the worker nodes, joining it to the cluster. 195 | 196 | ### Engage kubectl proxy 197 | The **kubectl** CLI includes a proxy that can be used to facilitate connections to the Kubernetes master node by remote machines, providing a convenient way of enabling: 198 | 199 | - Browser access to the Kubernetes Dashboard from a remote machine 200 | - Access by Opsview Monitor for monitoring 201 | 202 | To engage the proxy enabling both these things, log into the **k8smaster** node ... 203 | 204 | ``` 205 | ssh k8suser@k8smaster 206 | ``` 207 | 208 | ... and execute **kubectl proxy** as follows: 209 | 210 | ``` 211 | kubectl proxy --port=8080 --address='0.0.0.0' --accept-hosts='^*$' 212 | ``` 213 | 214 | This permits access to the Kubernetes master's entrypoint on port 8080 from machines on the local network. 215 | 216 | Killing the kubectl proxy is done by finding the PID of the running kubectl process on the master node: 217 | 218 | ``` 219 | ps -ef | grep kubectl 220 | ``` 221 | And then executing: 222 | 223 | ``` 224 | sudo kill -9 225 | ``` 226 | 227 | ### Dashboard Access 228 | Accessing the Kubernetes Dashboard from your management or other VM requires a several-step process: 229 | 230 | 1. Engage **kubectl proxy** (see above) on the Kubernetes master to expose the cluster entrypoint on a port (e.g., 8080). 231 | 1. Port-forward from your management computer to provide a tunnel for the browser: 232 | 233 | ``` 234 | ssh -L 9000:localhost:8080 k8suser@k8smaster 235 | ``` 236 | ... where 9000 is an unused port. Thereafter, you can bring up the Dashboard on your management machine's browser by opening a tab to localhost:9000 on the following path: 237 | 238 | ``` 239 | http://localhost:9000/api/v1/namespaces/kube-system/services/https:kubernetes-dashboard:/proxy/#!/login 240 | ``` 241 | 242 | ###### Authenticating to the Kubernetes Dashboard 243 | To log into the Kubernetes Dashboard, the easiest method is to use bearer token authentication. The Ansible scripts retrieve a JSON payload containing (among other information) this bearer token for your administrative user, and save it in a local file: **kubernetes-ansible-example/current_cluster/bearer-token.txt** 244 | 245 | To retrieve the bearer token, it's helpful to use **jq** an open source Linux JSON parser. This can be installed on Ubuntu as follows: 246 | 247 | ``` 248 | sudo apt-get install jq 249 | ``` 250 | Thereafter, you can switch to the current_cluster directory and extract the bearer token: 251 | 252 | ``` 253 | cd current_cluster 254 | cat bearer-token.txt | jq ".stdout_lines[12]" 255 | ``` 256 | The token, a long hex string, can then be copied into the appropriate field of the Dashboard's login dialog. 257 | 258 | See the [Kubernetes Dashboard Guide](https://github.com/kubernetes/dashboard/wiki/Creating-sample-user) for more. 259 | 260 | ### Flight Check 261 | Some basic flight checks will confirm the health of your Kubernetes cluster. ssh to the master node with: 262 | 263 | ``` 264 | ssh k8suser@k8smaster 265 | ``` 266 | 267 | And try some **kubectl** commands (Examples): 268 | 269 | ``` 270 | kubectl get pods --all-namespaces (lists all running pods) 271 | ``` 272 | ``` 273 | kubectl cluster-info 274 | ``` 275 | 276 | ### Resources 277 | 278 | [Kubernetes basics](https://kubernetes.io/docs/tutorials/kubernetes-basics/) 279 | [kubectl Overview](https://kubernetes.io/docs/reference/kubectl/overview/) 280 | [Kubernetes Dashboard Guide](https://github.com/kubernetes/dashboard/wiki/Creating-sample-user) 281 | 282 | ### Opsview Tutorials 283 | Tutorials based around this Kubernetes cluster will be posted at [Opsview.com](https://www.opsview.com/resources/blog) 284 | -------------------------------------------------------------------------------- /roles/configure-nodes/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: create .exrc idempotent 3 | copy: content="set nocompatible\nset backspace=2\n" dest=.exrc 4 | 5 | - name: turn swap off 6 | command: swapoff -a 7 | 8 | - name: comment out swap line in /etc/fstab idempotent 9 | lineinfile: 10 | path: /etc/fstab 11 | backup: yes 12 | backrefs: yes 13 | regexp: '(^(?!#).*swap *sw.*$)' 14 | line: '# \1' 15 | 16 | - name: reboot server 17 | shell: 'sleep 1 && shutdown -r now "Rebooting to ensure swap is off" && sleep 1' 18 | async: 1 19 | poll: 0 20 | become: true 21 | 22 | - name: Wait for server to restart 23 | wait_for_connection: 24 | 25 | - name: Pause a little more 26 | pause: 27 | minutes: 1 28 | 29 | - name: install libgetopt-mixed-perl for opsview agent 30 | apt: 31 | name: libgetopt-mixed-perl 32 | state: present 33 | update_cache: yes 34 | tags: 35 | - install_opsview_agent_pre_1 36 | 37 | - name: install libmcrypt4 for opsview agent 38 | apt: 39 | name: libmcrypt4 40 | state: present 41 | update_cache: yes 42 | tags: 43 | - install_opsview_agent_pre_2 44 | 45 | - name: install opsview agent 1 46 | get_url: 47 | url: https://s3.amazonaws.com/opsview-agents/ubuntu16/opsview-agent_5.4.1.172541017-1xenial1_amd64.deb 48 | dest: . 49 | tags: 50 | - install_opsview_agent_1 51 | 52 | - name: install opsview agent 2 53 | command: dpkg -i ./opsview-agent_5.4.1.172541017-1xenial1_amd64.deb 54 | register: out 55 | tags: 56 | - install_opsview_agent_2 57 | 58 | - debug: var=out.stdout_lines 59 | 60 | - name: install opsview agent 3 61 | command: apt-get install -f 62 | args: 63 | warn: no 64 | register: out 65 | tags: 66 | - install_opsview_agent_3 67 | 68 | - debug: var=out.stdout_lines 69 | 70 | - name: Install docker.io 71 | apt: 72 | name: docker.io 73 | state: present 74 | update_cache: yes 75 | 76 | - name: rewrite /lib/systemd/system/docker.service 77 | lineinfile: 78 | path: /lib/systemd/system/docker.service 79 | backup: yes 80 | backrefs: yes 81 | regexp: '(^ExecStart((?!-H tcp:\/\/0\.0\.0\.0:4243).)*)(\$DOCKER_OPTS$)' 82 | line: '\1 -H tcp://0.0.0.0:4243 \3' 83 | 84 | - name: reload the daemon 85 | command: systemctl daemon-reload 86 | 87 | - name: restart dockerd 88 | service: 89 | name: docker 90 | state: restarted 91 | 92 | - name: install libwww-perl 93 | apt: 94 | name: libwww-perl 95 | state: present 96 | 97 | - name: install apt-transport-https 98 | apt: 99 | name: apt-transport-https 100 | state: present 101 | 102 | - name: download and add Google GPG key 103 | apt_key: 104 | id: BA07F4FB 105 | url: https://packages.cloud.google.com/apt/doc/apt-key.gpg 106 | state: present 107 | 108 | - name: add Google repos to apt sources 109 | shell: 110 | cmd: | 111 | cat </etc/apt/sources.list.d/kubernetes.list 112 | deb http://apt.kubernetes.io/ kubernetes-xenial main 113 | EOF 114 | 115 | - name: install kubelet, kubeadm, and kubectl 116 | apt: 117 | force_apt_get: yes 118 | name: 119 | - kubelet 120 | - kubeadm 121 | - kubectl 122 | state: present 123 | update_cache: yes 124 | 125 | - name: configure Kubernetes cgroup driver to match Docker's (cgroupfs) 126 | lineinfile: 127 | path: /etc/systemd/system/kubelet.service.d/10-kubeadm.conf 128 | insertbefore: '^ExecStart' 129 | firstmatch: yes 130 | line: 'Environment="KUBELET_CGROUP_ARGS=--cgroup-driver=cgroupfs"' 131 | state: present 132 | 133 | - name: reload daemon 134 | command: systemctl daemon-reload 135 | 136 | - name: restart kubelet 137 | command: systemctl restart kubelet 138 | 139 | ... 140 | -------------------------------------------------------------------------------- /roles/create-k8s-master/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: reset Kubernetes master (just in case) 3 | command: kubeadm reset 4 | register: out 5 | 6 | - debug: var=out.stdout_lines 7 | 8 | - name: initialize Kubernetes master 9 | command: kubeadm init 10 | register: out 11 | 12 | - debug: var=out.stdout_lines 13 | 14 | - name: create local file with info returned by kubeadm init 15 | local_action: copy content="{{ out }}" dest="./current_cluster/kubeadm-init.txt" 16 | 17 | 18 | 19 | ... 20 | 21 | 22 | -------------------------------------------------------------------------------- /roles/install-weave/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - name: precondition for weave network 4 | command: sysctl net.bridge.bridge-nf-call-iptables=1 5 | register: out 6 | 7 | - debug: var=out.stdout_lines 8 | 9 | - name: set up regular kubectl user 1 10 | file: 11 | path: /home/k8suser/.kube 12 | state: directory 13 | mode: 0777 14 | tags: 15 | - setup_regular_user_1 16 | 17 | - name: set up regular kubectl user 2 18 | shell: cp -i /etc/kubernetes/admin.conf /home/k8suser/.kube/config 19 | tags: 20 | - setup_regular_user_2 21 | 22 | - name: set up regular kubectl user 3 23 | shell: chmod 777 /home/k8suser/.kube/config 24 | args: 25 | warn: false 26 | tags: 27 | - setup_regular_user_3 28 | 29 | - name: set up weave network 30 | shell: export KUBECONFIG=/etc/kubernetes/admin.conf && export kubever=$(kubectl version | base64 | tr -d '\n') && kubectl apply -f "https://cloud.weave.works/k8s/net?k8s-version=$kubever" 31 | register: out 32 | tags: 33 | - setup_weave_network_2 34 | 35 | - debug: var=out.stdout_lines 36 | ... 37 | -------------------------------------------------------------------------------- /roles/join-workers/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - name: join workers 4 | vars: 5 | join_command: "{{ lookup('file', './current_cluster/kubeadm-init.txt') }}" 6 | shell: "{{ join_command.stdout_lines[-1] }}" 7 | register: out 8 | 9 | - debug: var=out.stdout_lines 10 | 11 | ... 12 | -------------------------------------------------------------------------------- /roles/wrapup/files/xxx.yml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | name: admin-user 5 | namespace: kube-system 6 | -------------------------------------------------------------------------------- /roles/wrapup/files/yyy.yml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRoleBinding 3 | metadata: 4 | name: admin-user 5 | roleRef: 6 | apiGroup: rbac.authorization.k8s.io 7 | kind: ClusterRole 8 | name: cluster-admin 9 | subjects: 10 | - kind: ServiceAccount 11 | name: admin-user 12 | namespace: kube-system 13 | -------------------------------------------------------------------------------- /roles/wrapup/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: de-taint kubernetes master node for dashboard install 3 | shell: export KUBECONFIG=/etc/kubernetes/admin.conf && kubectl taint nodes --all node-role.kubernetes.io/master- 4 | ignore_errors: yes 5 | register: out 6 | 7 | - debug: var=out.stdout_lines 8 | 9 | - name: install kubernetes dashboard 10 | shell: export KUBECONFIG=/etc/kubernetes/admin.conf && kubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/master/src/deploy/recommended/kubernetes-dashboard.yaml 11 | register: out 12 | 13 | - debug: var=out.stdout_lines 14 | 15 | - name: copy xxx.yaml and yyy.yaml to server for creating admin user for bearer token access to dashboard 16 | copy: 17 | src: ./files/ 18 | dest: /home/k8suser 19 | 20 | - name: create admin user 1 21 | shell: export KUBECONFIG=/etc/kubernetes/admin.conf && kubectl create -f /home/k8suser/xxx.yml 22 | register: out 23 | 24 | - debug: var=out.stdout_lines 25 | 26 | - name: create admin user 2 27 | shell: export KUBECONFIG=/etc/kubernetes/admin.conf && kubectl create -f /home/k8suser/yyy.yml 28 | register: out 29 | 30 | - debug: var=out.stdout_lines 31 | 32 | - name: return bearer token 33 | shell: export KUBECONFIG=/etc/kubernetes/admin.conf && kubectl -n kube-system describe secret $(kubectl -n kube-system get secret | grep admin-user | awk '{print $1}') 34 | register: out 35 | 36 | - debug: var=out.stdout_lines 37 | 38 | - name: create local file with info returned by kubeadm init 39 | local_action: copy content="{{ out }}" dest="./current_cluster/bearer-token.txt" 40 | 41 | - name: set up docker not to require sudo 42 | shell: gpasswd -a k8suser docker 43 | register: out 44 | 45 | - debug: var=out.stdout_lines 46 | ... 47 | -------------------------------------------------------------------------------- /top.retry: -------------------------------------------------------------------------------- 1 | k8sworker1 2 | k8sworker2 3 | -------------------------------------------------------------------------------- /top.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - hosts: all 3 | become: true 4 | gather_facts: False 5 | tasks: 6 | - name: install python 2 7 | raw: test -e /usr/bin/python || (apt -y update && apt install -y python-minimal) 8 | 9 | - hosts: all 10 | become: true 11 | roles: 12 | - configure-nodes 13 | 14 | - hosts: k8smaster 15 | become: true 16 | roles: 17 | - create-k8s-master 18 | - install-weave 19 | - wrapup 20 | 21 | - hosts: 22 | - k8sworker1 23 | - k8sworker2 24 | become: true 25 | roles: 26 | - join-workers 27 | ... 28 | --------------------------------------------------------------------------------