├── .gitignore ├── .travis.yml ├── Gemfile ├── LICENSE ├── README.adoc ├── Vagrantfile ├── cheatsheet.adoc ├── clean.bat ├── complete-bootstrap.sh ├── mapfile-cloud.yml ├── mapfile-vagrant.yml ├── master └── srv │ ├── .gitignore │ ├── pillar │ ├── .gitignore │ ├── cloudcredentials.sls.sample │ └── top.sls │ └── salt │ ├── cloud │ ├── conf │ │ ├── .gitignore │ │ ├── centos-digitalocean.conf │ │ ├── centos-saltify.conf │ │ ├── cloud.conf │ │ ├── digitalocean.conf │ │ └── saltify.conf │ └── init.sls │ ├── consul-template │ ├── conf │ │ └── consul-template.service │ ├── init.sls │ └── templates │ │ └── template.conf │ ├── consul │ ├── conf │ │ ├── common.json │ │ ├── consul-ui-service.json │ │ ├── consul-ui.json │ │ ├── consul.json │ │ ├── consul.service │ │ ├── encrypt.json │ │ └── sysconfig-consul │ └── init.sls │ ├── dnsmasq │ ├── conf │ │ ├── dnsmasq-consul.conf │ │ └── networkmanager-dnsmasq.conf │ └── init.sls │ ├── fabio │ ├── conf │ │ ├── fabio.properties │ │ └── fabio.service │ └── init.sls │ ├── lb │ ├── conf │ │ ├── nginx-consul.conf │ │ ├── nginx.conf │ │ ├── nginx.json │ │ └── upstream.ctmpl │ └── init.sls │ ├── tools.sls │ ├── top.sls │ └── web │ ├── conf │ ├── httpd.json │ ├── index.html │ └── index.json │ └── init.sls ├── saltmaster.tf ├── saltmaster_override.tf.sample ├── setpath.bat └── tutorial ├── 10_introduction ├── index.adoc └── overview.puml ├── 20_getting_started ├── index.adoc ├── putty_keyauth.png └── puttygen_createkey.png ├── 30_setup_saltstack └── index.adoc ├── 35_setup_saltcloud └── index.adoc ├── 40_create_webservers └── index.adoc ├── 45_setup_monitoring ├── consul-ui.png └── index.adoc ├── 50_create_loadbalancer └── index.adoc ├── 60_cleanup └── index.adoc ├── Gemfile ├── build.bat ├── build.sh ├── setup.bat ├── tutorial.adoc └── update-gh-pages.sh /.gitignore: -------------------------------------------------------------------------------- 1 | /.vagrant 2 | /bin 3 | /download 4 | /terraform.tfstate.backup 5 | /terraform.tfstate 6 | /saltmaster_override.tf 7 | /.idea 8 | /tutorial/output 9 | /tutorial/10_introduction/images/uml-example.png 10 | /master.bat 11 | /sync.bat 12 | /id_rsa* -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: ruby 2 | rvm: 3 | - 1.9.3 4 | addons: 5 | apt: 6 | packages: 7 | - graphviz 8 | script: ./tutorial/build.sh 9 | after_success: ./tutorial/update-gh-pages.sh 10 | env: 11 | global: 12 | secure: molF4N6WO21tOQWsq2Wpj810UKJwqYf+IgltNjwLDJ6GG42o0w1Ae4mNc6NQ4FPngF7Tl4x8LqJbo9rRjdS1ZblAz7ARP4Y2s41ClKA3q4jNMS2/1yKKCX/4YqKsbwi+pOZ66/908MDqih1khB0nvJtA9RHeSbVFs1q1OyonWEUmUasP4PXR9U9y8UV+KVFIP/+NcOsCNEoHfVSNqAp1RAOobPsHd72CJ6eq06XhlcB/BpBYgIQ6UQm1cU1sB74PNida82TjJqN0uebOvb/gqCPM01Br1PDktgHa2Sznc1VlcMhicLkj9gL/MPiudwkpwtsrzaV8nnq6ewAZoKPsqtNYZ7CdsnogbUKf4owMjRWx6muGSt9+rKMW6LzZutBetCAWMnTTaxbni6FjwC2cnpkgyDohcBVl60Ei8oCXUHIq+F2K+mQASSy2skz8sV17qBx5sqmHFmCVvVBmlstmVnDXBSVbX8kTDq+WHlTOHlm2eZscHNKLZqkjsyweHCWd7v0t3rwXqN8YgWXq/PJp8kbkqYE12UXOyEHDn9m0EzLUGk/QQf/zZJERzj8/e0TZffWLoZSs2iHJH1qJE/0lHEU+/udAQvc5BbEEQCuEnKuRQY+9FOaja0cO1rKf2nPdzA9UC65FZQwIO8mBLiN0i9mIs+bIbSZpAc3DUw0t66Y= 13 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | # this is used by Travis CI to build the online manual 2 | source 'https://rubygems.org' 3 | gem 'asciidoctor' 4 | gem 'coderay' 5 | gem 'asciidoctor-diagram' 6 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.adoc: -------------------------------------------------------------------------------- 1 | ifdef::env-github[:outfilesuffix: .adoc] 2 | :ext-relative: {outfilesuffix} 3 | 4 | # Example Project for Terraform, Salt, Consul and DigitalOcean 5 | 6 | image:https://travis-ci.org/ahus1/saltconsul-examples.svg["Build Status", link="https://travis-ci.org/ahus1/saltconsul-examples"] 7 | 8 | ## About 9 | 10 | This started out as an example on how to use the three software packages and a cloud provider in a tutorial: 11 | 12 | * http://terraform.io/[HashiCorp's Terraform^] - spin up new servers and orchestrate resources in the cloud from a simple text file. 13 | 14 | * http://docs.saltstack.com/[SaltStack's Salt^] - install and manage software on a server. With it's cloud add-on it can also spin up new servers in the cloud. 15 | 16 | * https://consul.io/[HashiCorp's Consul^] - service discovery and monitoring in the cloud. 17 | 18 | * https://www.digitalocean.com/[DigitalOcean^] - a cloud provider. Use their API to spin up virtual servers as needed. 19 | 20 | This setup will show you a setup where a ngix load balancer distributes the load between multiple web servers. 21 | 22 | There is a http://ahus1.github.io/saltconsul-examples/tutorial.html[full tutorial^] available online. 23 | The sources for this tutorial are in the `tutorial` sub folder of this project. 24 | 25 | For experienced users there is a <> in `cheatsheet.adoc`. 26 | 27 | ## License 28 | 29 | Copyright 2015 Alexander Schwartz 30 | 31 | Licensed under the Apache License, Version 2.0 (the "License"); 32 | you may not use this file except in compliance with the License. 33 | You may obtain a copy of the License at 34 | 35 | http://www.apache.org/licenses/LICENSE-2.0 36 | 37 | Unless required by applicable law or agreed to in writing, software 38 | distributed under the License is distributed on an "AS IS" BASIS, 39 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 40 | See the License for the specific language governing permissions and 41 | limitations under the License. 42 | -------------------------------------------------------------------------------- /Vagrantfile: -------------------------------------------------------------------------------- 1 | # -*- mode: ruby -*- 2 | # vi: set ft=ruby : 3 | 4 | # Vagrantfile API/syntax version. Don't touch unless you know what you're doing! 5 | VAGRANTFILE_API_VERSION = "2" 6 | 7 | Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| 8 | # All Vagrant configuration is done here. The most common configuration 9 | # options are documented and commented below. For a complete reference, 10 | # please see the online documentation at vagrantup.com. 11 | 12 | # Every Vagrant virtual environment requires a box to build off of. 13 | config.vm.box = "bento/centos-7.1" 14 | 15 | if Vagrant.has_plugin?("vagrant-cachier") 16 | config.cache.scope = :box 17 | end 18 | 19 | # Disable automatic box update checking. If you disable this, then 20 | # boxes will only be checked for updates when the user runs 21 | # `vagrant box outdated`. This is not recommended. 22 | config.vm.box_check_update = false 23 | 24 | # Provider-specific configuration so you can fine-tune various 25 | # backing providers for Vagrant. These expose provider-specific options. 26 | # Example for VirtualBox: 27 | # 28 | # config.vm.provider "virtualbox" do |vb| 29 | # # Don't boot with headless mode 30 | # vb.gui = true 31 | # 32 | # # Use VBoxManage to customize the VM. For example to change memory: 33 | # vb.customize ["modifyvm", :id, "--memory", "1024"] 34 | # end 35 | # 36 | # View the documentation for the provider you're using for more 37 | # information on available options. 38 | 39 | if Vagrant.has_plugin?("vagrant-vbguest") 40 | # https://github.com/dotless-de/vagrant-vbguest 41 | # set auto_update to false, if you do NOT want to check the correct 42 | # additions version when booting this machine 43 | config.vbguest.auto_update = false 44 | 45 | # do NOT download the iso file from a webserver 46 | config.vbguest.no_remote = true 47 | end 48 | 49 | ## VM COnfigurations 50 | 51 | 52 | # Deployment instance salt master 53 | config.vm.define :master do |master| 54 | master.vm.network :private_network, ip: "192.168.23.12" 55 | master.vm.hostname = 'master' 56 | 57 | master.vm.synced_folder "master/srv/", "/srv/" 58 | master.vm.network :forwarded_port, guest: 22, host: 2201, auto_correct: true 59 | 60 | # salt master ports 61 | master.vm.network :forwarded_port, guest: 4505, host: 4505 62 | master.vm.network :forwarded_port, guest: 4506, host: 4506 63 | 64 | # web server 65 | master.vm.network :forwarded_port, guest: 80, host: 8080 66 | 67 | # consul ports 68 | master.vm.network :forwarded_port, guest: 8300, host: 8300, protocol: 'udp' 69 | master.vm.network :forwarded_port, guest: 8301, host: 8301, protocol: 'udp' 70 | master.vm.network :forwarded_port, guest: 8302, host: 8302, protocol: 'udp' 71 | master.vm.network :forwarded_port, guest: 8300, host: 8300, protocol: 'tcp' 72 | master.vm.network :forwarded_port, guest: 8301, host: 8301, protocol: 'tcp' 73 | master.vm.network :forwarded_port, guest: 8302, host: 8302, protocol: 'tcp' 74 | 75 | # consul UI - http://localhost:8500/ui 76 | master.vm.network :forwarded_port, guest: 8500, host: 8500, protocol: 'tcp' 77 | 78 | master.vm.provider "virtualbox" do |v| 79 | v.name = "master" 80 | end 81 | 82 | # networking problems with CentOS 7.1 83 | # https://github.com/mitchellh/vagrant/issues/5590 84 | master.vm.provision "shell", inline: "nmcli connection reload; systemctl restart network.service" 85 | 86 | master.vm.provision :salt do |config| 87 | config.install_master = true 88 | config.verbose = true 89 | # show debug output, don't start servers, look for master on localhost 90 | config.bootstrap_options = "-D -X -A localhost" 91 | config.temp_config_dir = "/tmp" 92 | end 93 | # initial provisioning for problem above. 94 | 95 | $sshkey = < 2 | 3 |

Hello world!

4 |

This is 5 | 6 | {% if salt['grains.get']('ip4_interfaces:eth0', None) %} 7 | {{ grains['ip4_interfaces']['eth0'][0] }} 8 | {% endif %} 9 | 10 | {% if salt['grains.get']('ip4_interfaces:enp0s8', None) %} 11 | {{ grains['ip4_interfaces']['enp0s8'][0] }} 12 | {% endif %} 13 | 14 | talking!

15 | 16 | -------------------------------------------------------------------------------- /master/srv/salt/web/conf/index.json: -------------------------------------------------------------------------------- 1 | { "ip": 2 | {% if salt['grains.get']('ip4_interfaces:eth0', None) %} 3 | "{{ grains['ip4_interfaces']['eth0'][0] }}" 4 | {% endif %} 5 | 6 | {% if salt['grains.get']('ip4_interfaces:enp0s8', None) %} 7 | "{{ grains['ip4_interfaces']['enp0s8'][0] }}" 8 | {% endif %} 9 | } -------------------------------------------------------------------------------- /master/srv/salt/web/init.sls: -------------------------------------------------------------------------------- 1 | # tag::websetup[] 2 | # install the web server package 3 | httpd: 4 | pkg.installed: 5 | - name: httpd 6 | service.running: 7 | - enable: true 8 | 9 | /var/www/html/index.html: 10 | file.managed: 11 | - template: jinja 12 | - source: salt://web/conf/index.html 13 | /var/www/html/index.json: 14 | file.managed: 15 | - template: jinja 16 | - source: salt://web/conf/index.json 17 | # end::websetup[] 18 | 19 | # tag::consul[] 20 | /etc/consul/httpd.json: 21 | file.managed: 22 | - template: jinja 23 | - source: salt://web/conf/httpd.json 24 | - watch_in: 25 | - service: consul 26 | # end::consul[] 27 | -------------------------------------------------------------------------------- /saltmaster.tf: -------------------------------------------------------------------------------- 1 | # Configure the DigitalOcean Provider 2 | 3 | # tag::setupkey[] 4 | # setup connection to digitalocen using the API token 5 | provider "digitalocean" { 6 | token = "cb93...f552" 7 | } 8 | 9 | # Fingerprint of the key, just to have it at hand. Please exchange it with your key 10 | # "c8:df:df:...:d9:41:ce" 11 | # public DigitalOcean SSH key 12 | resource "digitalocean_ssh_key" "default" { 13 | name = "Terraform Example" 14 | public_key = "${file("id_rsa.pub")}" 15 | } 16 | 17 | # setup one small virtual server in Frankfurt. 18 | # tag::setupsaltstart[] 19 | resource "digitalocean_droplet" "master" { 20 | # end::setupsaltstart[] 21 | image = "centos-7-0-x64" 22 | name = "master" 23 | region = "nyc2" 24 | size = "512mb" 25 | 26 | # install this SSH key on the machine so we can access it later 27 | ssh_keys = ["c8:df:df:...:d9:41:ce"] 28 | 29 | # ensure that ssh key is available before we setup this machine 30 | # as in terraform 0.4.2 there is no automatic dependency here (TODO) 31 | depends_on = [ "digitalocean_ssh_key.default" ] 32 | 33 | # end::setupkey[] 34 | 35 | # tag::setupsalt[] 36 | connection { 37 | type = "ssh" 38 | host = "${digitalocean_droplet.master.ipv4_address}" 39 | port = 22 40 | timeout = "5m" 41 | user = "root" 42 | key_file = "id_rsa" 43 | } 44 | 45 | provisioner "local-exec" { 46 | command = "echo bin\ssh root@${digitalocean_droplet.master.ipv4_address} -i id_rsa > master.bat " 47 | } 48 | 49 | provisioner "local-exec" { 50 | command = "echo bin\rsync -vr -e 'ssh -i id_rsa' master/srv root@${digitalocean_droplet.master.ipv4_address}:/ > sync.bat " 51 | } 52 | 53 | # ensure that SSH Keys are copied to the salt master in the next step 54 | provisioner "local-exec" { 55 | command = "if not exist master\srv\salt\cloud\conf (mkdir master\srv\salt\cloud\conf)" 56 | } 57 | provisioner "local-exec" { 58 | command = "copy /y id_rsa* master\srv\salt\cloud\conf" 59 | } 60 | 61 | # copy copntents of master/srv to the server 62 | provisioner "file" { 63 | source = "master/srv" 64 | destination = "/" 65 | } 66 | 67 | 68 | # copy file with additional provisioning commands to the server 69 | provisioner "file" { 70 | source = "complete-bootstrap.sh" 71 | destination = "/tmp/complete-bootstrap.sh" 72 | } 73 | 74 | # install salt minion and master 75 | provisioner "remote-exec" { 76 | inline = [ 77 | # install salt-minion and salt-master, but don't start services 78 | "curl -L https://bootstrap.saltstack.com | sh -s -- -M -X -A localhost", 79 | # work around possible missing executable flag 80 | "cat /tmp/complete-bootstrap.sh | sh -s" 81 | ] 82 | } 83 | 84 | # end::setupsalt[] 85 | 86 | # tag::setupkey[] 87 | } 88 | # end::setupkey[] 89 | -------------------------------------------------------------------------------- /saltmaster_override.tf.sample: -------------------------------------------------------------------------------- 1 | # tag::setupkey[] 2 | provider "digitalocean" { 3 | token = "cb93..f552" 4 | } 5 | 6 | resource "digitalocean_droplet" "master" { 7 | ssh_keys = ["c8:df:...:41:ce"] 8 | } 9 | # end::setupkey[] 10 | -------------------------------------------------------------------------------- /setpath.bat: -------------------------------------------------------------------------------- 1 | SET PATH=bin;%PATH% 2 | -------------------------------------------------------------------------------- /tutorial/10_introduction/index.adoc: -------------------------------------------------------------------------------- 1 | [[introduction]] 2 | ## Introduction 3 | 4 | This is a tutorial to show a step by step example for an initial setup in the cloud. By following this guide step by step you'll have a service running in the cloud at the end. 5 | 6 | If you have questions please contact me (alexander.schwartz@gmx.net), discuss it on GitHub (https://github.com/ahus1/saltconsul-examples/) or even submit pull requests. 7 | You can meet me in person at conferences and user group meetings. 8 | 9 | ### Szenario 10 | 11 | The following scenario will be implemented step by step: A user will use his or her browser to contact a load balancer. This load balancer will know about two web servers and will forward the requests accordingly. See the following picture for details. 12 | 13 | 14 | .Infrastructure Big Picture 15 | ifdef::env-browser[] 16 | [uml,file="overview.png"] 17 | endif::env-browser[] 18 | ifndef::env-browser[] 19 | ["plantuml", "overview", "png"] 20 | endif::env-browser[] 21 | -- 22 | include::overview.puml[] 23 | -- 24 | 25 | ### Tools and providers we use 26 | 27 | #### DigitalOcean 28 | 29 | There are multiple hosting providers in the cloud. A very simple configuration for a start can be found at https://www.digitalocean.com/?refcode=532ccb598c03[DigitalOcean^]. 30 | 31 | They offer a server starting from 5 USD a month with hourly billing. All functionality is available using an API. DigialOcean has been integrated in several automation tools already. 32 | 33 | #### Hashicorp's Terraform 34 | 35 | The first step is to create a first server in the cloud with an initial setup. A tool that is suited for this https://www.terraform.io/[Terraform^]. 36 | 37 | It allows you to define the resources of your cloud infrastructure in a short and concise format in a text file. 38 | You can use it to create and update your infrastructure in the cloud. 39 | 40 | #### Saltstack's Salt 41 | 42 | Once the server has been set up, it needs to be "provisioned", meaning that the all software and configuration settings are being applied. There are tools like Chef, Puppet and Ansible for this. A very recent tool to do this is http://docs.saltstack.com[Salt^]. 43 | 44 | Again, this allows us to define the resource of the cloud infrastructure in a short an concise format in a text file. 45 | You can use it to create, update and manage your infrastructure in the cloud. 46 | 47 | #### Hashicorp's Consul 48 | 49 | Once all the servers have been set up and provisioned, they need to know about each other, and they need to be monitored. 50 | A tool that is well suited for cloud environments like this is https://consul.io/[Consul^]. 51 | 52 | It runs as a sidecar next to each service and monitors the service and the host it is running on. 53 | 54 | Using Consul the load balancer will be aware of the different web servers it can forward requests to. 55 | 56 | #### CentOS Linux 57 | 58 | The Linux distribution used here is https://www.centos.org/[CentOS Linux Version 7^]. 59 | It is derived from the sources of RedHat Enterprise Linux and is freely available. 60 | The new servers that we spin up here are all based on this distribution. 61 | I have chosen this Linux distribution as RedHat Enterprise Linux and CentOS are very present in the enterprises I have worked for. 62 | CentOS might not contain the very latest versions of several software packages, nevertheless this tutorial shows that all this works as well. 63 | It will also work with different flavors of Linux, although some of the names of the packages that are installed here will have different names. 64 | Consul and Salt will also work on Windows machines. 65 | 66 | #### Apache httpd 67 | 68 | http://httpd.apache.org/[Apache httpd^] is used as a web server in this setup. 69 | It will only serve static pages. 70 | You are free to extend this setup to enhance it to serve dynamic pages i.e. with PHP, or exchange it with an application server like Apache Tomcat. 71 | The load balancer will still work as expected. 72 | 73 | #### NGINX 74 | 75 | http://nginx.com/[NGINX^] is a web server that can be used to serve static pages, but it is also very popular to act as reverse proxy. 76 | As a reverse proxy it is the contact for the users on the internet. It forwards all incoming requests to backend web servers. -------------------------------------------------------------------------------- /tutorial/10_introduction/overview.puml: -------------------------------------------------------------------------------- 1 | @startuml 2 | 3 | 4 | actor User 5 | 6 | node "Load Balancer" { 7 | 8 | [nginx] 9 | 10 | [consul] as consullb 11 | 12 | } 13 | 14 | node "Webserver 1" { 15 | 16 | [consul] as consulws1 17 | 18 | [httpd] as httpdws1 19 | 20 | } 21 | 22 | node "Webserver 2" { 23 | 24 | [consul] as consulws2 25 | 26 | [httpd] as httpdws2 27 | 28 | } 29 | 30 | consullb .. consulws2 31 | consullb .. consulws1 32 | 33 | nginx --> httpdws1 34 | nginx --> httpdws2 35 | 36 | User --> nginx 37 | 38 | @enduml 39 | -------------------------------------------------------------------------------- /tutorial/20_getting_started/index.adoc: -------------------------------------------------------------------------------- 1 | // using experimental to support btn: macro 2 | :experimental: 3 | 4 | ## Getting started 5 | 6 | In order to get started you'll need to register yourself at the cloud provider DigitalOcean and create yourself a SSH key. 7 | 8 | ### Project's directory 9 | 10 | Please create an empty directory on your hard drive. Please place all files you create inside this directory or in sub directories. 11 | 12 | Please avoid blanks or other special characters in the path name. This might confuse some programs. 13 | 14 | ### Setup Git 15 | 16 | It's a good practice to save intermediate states when you work with source code files. To do this it is very handy to use git. 17 | 18 | Please install https://tortoisegit.org/[TortoiseGit^] to do this. Please choose "Git Create repository here..." from your Explorer's context menu to setup a local Git repository. This will not be shared with anybody. But it allows you to go back to any previous step during this tutorial. Please commit the current state of your work after every step to be able to do this. 19 | 20 | ### Create a SSH key 21 | 22 | When you want to access a command line on a Linux server, you'll use secure shell (SSH) to access it. It is possible to use a username and password combination to log in. The more secure way is to use a public/private key pair. This has the following advantages: 23 | 24 | . A private key is _very_ long, and therefore it can't be guessed. 25 | 26 | . You can give away your public key to multiple servers to log in. Even if one of the servers is being compromised no-one will be able to impersonate yourself with the public key. 27 | 28 | To do that please download http://www.chiark.greenend.org.uk/~sgtatham/putty/download.html[PuTTY and PuTTYgen^]. Even if you already have a key pair, use the time to create a key pair just for this tutorial. 29 | 30 | Once you run it, you'll see the following dialoge shown in <>. 31 | 32 | [[img-puttygen-createkey]] 33 | image::puttygen_createkey.png[title="Creating a key with puttygen"] 34 | 35 | Please note the following settings: 36 | 37 | . The key type needs to be set to `SSH-2 RSA`. DSA will not work as it is not supported by DigitalOcean. 38 | 39 | . The number of bits in a generated key should be set to 2048. This key length will provide a very good level of security. 40 | 41 | Press btn:[Generate] and follow the instruction on the screen to create your key. 42 | 43 | You'll now need to save your private and your public key. 44 | 45 | There are different formats to save your keys. In these examples you'll need the OpenSSH format. 46 | This tutorial will not use a password for your keys. 47 | 48 | Please proceed as follows: 49 | 50 | * create a local file called `id_rsa.pub` in the project folder. Copy the contents of the field labeled "Public key for pasting into ..." into this file. It should be one long line that looks like this: 51 | 52 | ----- 53 | ssh-rsa AAAAB3NzaC1yc2E...ANqTeH4scPYaQ== rsa-key-20150527 54 | ----- 55 | 56 | * Choose menu:Conversions[Export OpenSSH key] to export your private key in the OpenSSH format. Choose `id_rsa` as a filename and save it the project folder. It should be multiple lines that that looks like this: 57 | 58 | --------- 59 | -----BEGIN RSA PRIVATE KEY----- 60 | MIIEoQIBAAKCAQEAxet1I80forMqWAj1gCexHQLoikdlMjUl8vM498vdfqrO1dHE 61 | ... 62 | ZCRXXL2Q5mlJDSekPLRnrgPervaWBVW8v9Bqgybp9qIigDAQxA== 63 | -----END RSA PRIVATE KEY----- 64 | --------- 65 | 66 | * Choose btn:[Save private key] to export your private key. Choose `id_rsa.ppk` as a filename and save it the project folder. It should be multiple lines that that looks like this: 67 | 68 | --------- 69 | PuTTY-User-Key-File-2: ssh-rsa 70 | ... 71 | Private-MAC: da8565a7228c35643a0047b001eafafb496e70f1 72 | --------- 73 | 74 | * Please also save the fingerprint in a separate file. You'll need that fingerprint later. Please save it in a file called `id_rsa.fingerprint`. It should look like this: 75 | 76 | --------- 77 | ssh-rsa 2048 c8:df:df:...:83:d9:41:ce 78 | --------- 79 | 80 | ### Register at DigitalOcean 81 | 82 | In order to run your virtual machines in the cloud you'll need to set up an account with a hosting provider. 83 | DigitalOcean is used in this example. It specializes in running virtual machines in the cloud. For this they have multiple data centers around the world. In April 2015 they have opened a data center in Frankfurt am Main, Germany. There are almost no additional services except hosting DNS entries. This will give us the clear view we'll need today. 84 | 85 | Please use the following afiliate link to register at DigitalOcean: https://www.digitalocean.com/?refcode=532ccb598c03. 86 | Using this link you'll automatically receive 10 USD credit. If you have registered before, or if you haven't received the credit, you could try to redeem `ALLSSD10` as a promo code to receive 10 USD credit. 87 | 88 | Once you have registered you should add some extra safety to your account by activating http://de.wikipedia.org/wiki/Google_Authenticator[Google Authenticator^]. When you activate it, you will need in addition to username and password also a token that is created by an app on your mobile phone. This way no-one can log in to your DigitalOcean account with only your username and password. 89 | 90 | In the https://cloud.digitalocean.com/settings/security[security settings of DigitalOcean^] you should set up two-factor-authentication. You'll need to https://support.google.com/accounts/answer/1066447[install Google Authenticator on a mobile device^] to create a PIN code whenever you log in to your DigitalOcean account. 91 | 92 | After you have registered you'll need to register a payment method, for example a credit card. This is a matter of authenticating yourself. 93 | 94 | As we'll want to script everything we do in the following steps, please create an API token. This needs to be a token with read and write access. You'll pass on this token to all programs that will access your account. At any point in time you can revoke it without changing your main password. 95 | 96 | // TODO: add link to API key creation 97 | 98 | ### Preparations for Terraform 99 | 100 | The next tool we'll use will output ANSI escape characters on your screen. While Linux terminals support it, Windows doesn't. To fix this please https://github.com/adoxa/ansicon/releases[download the ANSICON tool^]. 101 | 102 | Once downloaded, extract the contents to a `bin` subfolder of your project's directory. Open a command shell and run `bin\ansicon` once to enable ANSI transformations. 103 | 104 | ### Setup of Terraform and creating a first virtual machine 105 | 106 | Terraform can be used to orchestrate multiple resources in the cloud. A text file is used to describe the configuration you want to have and you can then set up of tear down the whole setup in one step. 107 | 108 | Terraform is distributed as a set of binaries. When you're running Windows, download `terraform_0.4.2_windows_amd64.zip` from Terraform's website https://www.terraform.io/. 109 | 110 | Extract all files to a subdirectory called `bin` in your project's folder. Please open a command line in your project's folder and run `bin\terraform`. This will print out short usage instructions on how to use it. 111 | 112 | Please create a first configuration file for Terraform called `saltmaster.tf`. In this file we'll put all the information necessary to setup the first host in the cloud. 113 | 114 | [source,json] 115 | .saltmaster.tf 116 | ---- 117 | include::../../saltmaster.tf[tags=setupkey] 118 | ---- 119 | 120 | Side note: if you want to put your secret and personal information in a separate file, please see the https://www.terraform.io/docs/configuration/override.html[override configuration of Terraform^]. This allows you to have a separate file `saltmaster_override.tf` with all the secret stuff that you can avoid to check in to a repository. 121 | 122 | [source,json] 123 | .saltmaster_override.tf 124 | ---- 125 | include::../../saltmaster_override.tf.sample[tags=setupkey] 126 | ---- 127 | 128 | Now run `bin\terraform plan` on the command line. When everything in your file is spelled correctly, Terraform prints a plan of the steps it will perform to create the described infrastructure for you. 129 | 130 | Once you are happy with it, run `bin\terraform apply`. 131 | 132 | You will see the output of Terraform on the console. When you check DigitalOcean's web console, you'll see that the first "Droplet" has been created. In your account settings you also see that a SSH key has been registered with your account. 133 | 134 | You can see the IP address of the new server in the output of the Terraform command. At any time later you can look up the server DigitalOcean's web console, or use `bin\terraform show` to show the state of the currently created resources. 135 | 136 | Terraform has created a local file called `terraform.tfstate` that includes a machine readable description of the previously created resources. 137 | 138 | Use `terraform show` to get a list of all resources that have been created. 139 | If you want to re-create the server, you can use `bin\terraform taint ` and any point. 140 | To issue this for the machine we have just created use `bin\terraform taint digitalocean_droplet.master` 141 | When you the issue another `bin\terraform apply` command, the resource will be deleted and re-created. 142 | 143 | Perform the following steps to connect to your new newly created machine: 144 | 145 | . Start Putty. 146 | 147 | . In the first screen with the settings enter the IP address of your newly created machine in the cloud. 148 | 149 | . See <> on where enter your private key. Then click on btn:[Open] to connect to your machine. 150 | 151 | . Use `root` as the user name when you log in. You should not be asked for a password if everything has been setup correctly. 152 | 153 | [[img-putty-keyauth]] 154 | image::putty_keyauth.png[title="Login with a private Putty key"] 155 | 156 | Congratulations! You have now set up automatically a server in the cloud. And as everything has been scripted, it is totally reproducible. Try out yourself with different combinations of `apply`, `taint` and `destroy` to get yourself familiar with this setup. -------------------------------------------------------------------------------- /tutorial/20_getting_started/putty_keyauth.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ahus1/saltconsul-examples/e599b5fc8970168128d2d9ed6140bd6d8d19772c/tutorial/20_getting_started/putty_keyauth.png -------------------------------------------------------------------------------- /tutorial/20_getting_started/puttygen_createkey.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ahus1/saltconsul-examples/e599b5fc8970168128d2d9ed6140bd6d8d19772c/tutorial/20_getting_started/puttygen_createkey.png -------------------------------------------------------------------------------- /tutorial/30_setup_saltstack/index.adoc: -------------------------------------------------------------------------------- 1 | ## Setup of Salt 2 | 3 | ### Startup the salt master in the cloud 4 | 5 | Terraform is a great tool to setup servers from scratch. This is why we used it in the previous example. But it has only limited capabilities to install software on the new server and to keep that up-to-date. 6 | 7 | We will apply the existing capabilities of Terraform to install a tool that is better suited to do that. This tool will be Salt. Like Terraform Salt takes the configuration of the new machine in a declarative form. Salt competes with Puppet, Chef and Ansible to install and configure software. Salt has been chosen as it support a very declarative syntax to define the state of software and configuration files. It is also known to scale up to many thousands of servers. 8 | 9 | First you'll need to tell Terraform how to connect to the new server. Do this by adding a `connection` to your Terraform configuration file. 10 | 11 | A set of configuration files will be needed on the Salt master, both in `/etc/salt` (for configuration files) and `/srv` (for state files describing the to-be state of your upcoming setup). It will also copy the SSH key you've created to the cloud as we'll use it to setup the additional servers. 12 | 13 | There is a procedure to boostrap salt on an empty machine that can be run as a single line of code on the command line. A second command runs the provisioning of the master. 14 | 15 | [source] 16 | .saltmaster.tf 17 | ---- 18 | include::../../saltmaster.tf[tags=setupsaltstart] 19 | ... 20 | include::../../saltmaster.tf[tags=setupsalt] 21 | ... 22 | } 23 | ---- 24 | 25 | Use the terraform commands `taint` and `apply` to recreate your server in the cloud. You'll now have a running salt master! This will be the command and control server we'll use in the following steps to setup the full cluster presented in the chapter <>. 26 | 27 | Besides installing salt on this master server, salt the defined configuration for the master server as the `minion.conf` contained `startup_states: highstate`. 28 | 29 | Please use SSH to log in to the server and have a look at the files. 30 | A `master.bat` file has been created automatically with the master's most recent IP address. 31 | 32 | 33 | 34 | Issue the following command to check if everything worked fine: 35 | 36 | ---- 37 | # ensure that the minion on the master connected successfully 38 | salt '*' test.ping 39 | ---- 40 | 41 | 42 | 43 | ---- 44 | # run a highstate command to double-check everything worked 45 | salt '*' state.highstate 46 | ---- 47 | 48 | ### Synchronize a local configuration to your salt master 49 | 50 | You can continue editing the files in the local `master/srv` folder and sync it to the salt master. 51 | 52 | Download https://www.itefix.net/content/cwrsync-free-edition[rsync for windows^] and unpack it to the `bin` folder. 53 | 54 | Use the `sync.bat` file to sync any changes you have made to the `srv` folder to the salt master. 55 | 56 | 57 | 58 | -------------------------------------------------------------------------------- /tutorial/35_setup_saltcloud/index.adoc: -------------------------------------------------------------------------------- 1 | ## Setup of Salt Cloud 2 | 3 | Now it's time to teach Salt how to spin up more servers. 4 | 5 | The start of the Salstack configuration is the file `top.sls` in the folder `salt`: 6 | 7 | [source,yaml] 8 | .salt/top.sls 9 | ---- 10 | include::../../master/srv/salt/top.sls[tags=mastersetup] 11 | ---- 12 | 13 | This should be read as follows: For the `base` configuration, when on the server `master`, please make sure that the state defined in `salt.cloud` is applied, when the `state.highstate` command is issued. The 14 | 15 | The state `salt.cloud` in this case is the file `cloud.sls` in the directory `salt`. 16 | The syntax is that different tokens are delimited with a dot ("`.`"). The elements at the beginning are directories. The last element can be a directory or a file. If it is a directory, Salt will look up the `init.sls` file. If it is a file, Salt will look for a file with the token name plus `.sls` at the end. 17 | 18 | [source,yaml] 19 | .salt/cloud/init.sls 20 | ---- 21 | include::../../master/srv/salt/cloud/init.sls[tags=saltintro] 22 | ---- 23 | 24 | Here are the contents of the files referenced: 25 | 26 | [source,yaml] 27 | .salt/cloud/conf/digitalocean.conf 28 | ---- 29 | include::../../master/srv/salt/cloud/conf/digitalocean.conf[tags=saltintro] 30 | ---- 31 | 32 | [source,yaml] 33 | .salt/cloud/centos-digitalocean.sls 34 | ---- 35 | include::../../master/srv/salt/cloud/conf/centos-digitalocean.conf[tags=saltintro] 36 | ---- 37 | 38 | 39 | As Saltstack wil setup some new servers with DigitalOcean for you, it will also need your access token like Terraform did. 40 | 41 | Environment specific information is being held in a pillar. Please place it within a file called `cloudcredentials.sls` within the `srv/pillar` directory. 42 | 43 | [source,yaml] 44 | .pillar/cloudcredentials.sls 45 | ---- 46 | include::../../master/srv/pillar/cloudcredentials.sls.sample[tags=saltintro] 47 | ---- 48 | 49 | You then reference this file in the `top.sls` file in the `srv/pillar` directory. 50 | 51 | [source,yaml] 52 | .pillar/top.sls 53 | ---- 54 | include::../../master/srv/pillar/top.sls[tags=saltintro] 55 | ---- 56 | 57 | This configuration ensures, that the secret API key is only available on the master. 58 | 59 | The difference between the salt states and the pillars is as follows: the contents of all states are available to all minions of a Salt master. 60 | But the contents of the pillars are delivered to the minions on a need-to-know basis. 61 | Therefore pillars will contain the secret bits of the configuration: passwords and keys. 62 | 63 | You can test this as follows: 64 | 65 | [source,shell] 66 | ---- 67 | # show list of available commands of salt-cloud 68 | salt-cloud -h 69 | # show the setup of providers (reads local configuration only) 70 | # should show my-digitalocean-config with digitalocean 71 | salt-cloud --list-providers 72 | # show the setup of profiles with digitalocean (ready local configuration only) 73 | # should show centos-digitalocean with my-digitalocean-config 74 | salt-cloud --list-profiles=my-digitalocean-config 75 | # show available locations for provider digitalocean 76 | salt-cloud --list-locations=my-digitalocean-config 77 | # show a list of instances in the cloud (accesses the cloud and uses the key, should be empty now) 78 | salt-cloud -Q 79 | ---- 80 | 81 | 82 | -------------------------------------------------------------------------------- /tutorial/40_create_webservers/index.adoc: -------------------------------------------------------------------------------- 1 | ## Create the webservers 2 | 3 | Now we have our Salt master running in the cloud as a command and control server. Using this as a stepping stone, we'll spin up some web servers. 4 | 5 | ### Start new nodes using Salt 6 | 7 | Now you can tell Salt to spin up new servers. This is done using the command 8 | 9 | ----- 10 | salt-cloud -l debug -p centos-digitalocean web1 11 | ----- 12 | 13 | You will see that Salt talks to the DigitalOcean API and creates a VM like Terraform has done before. 14 | The benefit is, that the new VM (a "minion") is controlled by our master server. 15 | 16 | Use the command `salt-key` you'll see all minions that are registered with our master. 17 | 18 | Using `salt '*' test.ping` the master shows all servers that are reachable. 19 | 20 | With `salt-cloud -Q` you'll see up-to-date information about the currently running servers running in the cloud. 21 | 22 | ### Install software on newly created nodes 23 | 24 | Now we'll define a state to describe what a web server should do. As a very simple example it will contain an Apache httpd web server and a simple HTML page that is being served. It will contain a little bit of dynamic content so that we'll know which server create the page once we have the load balancer in place. 25 | 26 | Create a file `web/init.sls` with the following content: 27 | 28 | [source,yaml] 29 | .web/init.sls 30 | ---- 31 | include::../../master/srv/salt/web/init.sls[tags=websetup] 32 | ---- 33 | 34 | This file will be evaluated on each web server using salt, and then saved as a static file in the httpd's folder. 35 | It will contain the public IP address of the server that is being retrieved from the Salt's context. 36 | 37 | [source,html] 38 | .web/conf/index.html 39 | ---- 40 | include::../../master/srv/salt/web/conf/index.html[] 41 | ---- 42 | 43 | 44 | Now tell Salt to run this state on all servers called `web*` that we will fire up in a second. 45 | 46 | [source,yaml] 47 | .top.sls 48 | ---- 49 | base: 50 | ... 51 | include::../../master/srv/salt/top.sls[tags=websetup] 52 | ---- 53 | 54 | To run this state on the the server start the command 55 | 56 | ---- 57 | salt '*' state.highstate 58 | ---- 59 | 60 | This will trigger all states on all registered servers. This will install the web server packages on the newly created server `web1`. 61 | 62 | Now point your browser to the IP address of your server in your browser. You'll now see the web page that we have deployed to the server. 63 | 64 | Let's install another server with just two more commands: 65 | 66 | ---- 67 | salt-cloud -l debug -p centos-digitalocean web2 68 | ---- 69 | 70 | As in the `digitalocean.conf` file contains the setting `startup_states: highstate`, the new server will be installed with the latest configuration settings on startup. 71 | -------------------------------------------------------------------------------- /tutorial/45_setup_monitoring/consul-ui.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ahus1/saltconsul-examples/e599b5fc8970168128d2d9ed6140bd6d8d19772c/tutorial/45_setup_monitoring/consul-ui.png -------------------------------------------------------------------------------- /tutorial/45_setup_monitoring/index.adoc: -------------------------------------------------------------------------------- 1 | ## Setup monitoring using Consul 2 | 3 | ### Basic setup of Consul 4 | 5 | Consul will do the monitoring and service discovery in our setup. 6 | It will know about all the web services that we have running, and monitors that they work properly. 7 | 8 | Consul consists of servers and agents. They differ only by configuration. 9 | In this setup there will be one server located on the salt master. 10 | In a production setup you'll want to have a high available cluster. 11 | 12 | Again we start with the `top.sls` configuration to register the new state. This time for all of our servers. 13 | 14 | [source,yaml] 15 | .salt/top.sls 16 | ---- 17 | base: 18 | ... 19 | include::../../master/srv/salt/top.sls[tags=consul] 20 | ---- 21 | 22 | [source,yaml] 23 | .salt/consul/init.sls 24 | ---- 25 | include::../../master/srv/salt/consul/init.sls[tags=consul] 26 | ---- 27 | 28 | [source,json] 29 | .salt/consul/conf/common.json 30 | ---- 31 | include::../../master/srv/salt/consul/conf/common.json[] 32 | ---- 33 | 34 | To show how to add monitoring consul will monitor its own web console using this service plus check. 35 | 36 | [source,json] 37 | .salt/consul/conf/consul-ui-service.json 38 | ---- 39 | include::../../master/srv/salt/consul/conf/consul-ui-service.json[] 40 | ---- 41 | 42 | Apply this configuration to you configuration by running: 43 | 44 | ---- 45 | salt '*' state.highstate 46 | ---- 47 | 48 | Salt will show you the applied settings. 49 | You can now look at consul's console using a web browser and the salt master's IP: http://:8500/. 50 | 51 | [[consul-ui]] 52 | image::consul-ui.png[title="Web UI of Consul"] 53 | 54 | ### Add monitoring for the web server 55 | 56 | As part of our web configuration please install an additional file `httpd.json` that will contain the service plus monitoring instructions. 57 | 58 | [source,yaml] 59 | .salt/web/init.sls 60 | ---- 61 | include::../../master/srv/salt/web/init.sls[tags=consul] 62 | ---- 63 | 64 | And here the file: 65 | 66 | [source,json] 67 | .salt/web/conf/httpd.json 68 | ---- 69 | include::../../master/srv/salt/web/conf/httpd.json[] 70 | ---- 71 | 72 | As soon as you deploy this configuration using Salt, you'll see that this service appears in the Consul UI. 73 | 74 | ---- 75 | salt '*' state.highstate 76 | ---- 77 | 78 | Congratulations! You now have monitoring up and running. And Consul knows about all the hosts 79 | that are running you web service. 80 | 81 | ### Using Consul for DNS resolving 82 | 83 | You can use `dnsmasq` to forward all DNS inquiries ending with `.consul` to consul. 84 | 85 | The state `dnsmasq/init.sls` will setup NetworkManager to use dnsmasq, and will setup dnsmasq to forward all requests to Consul. 86 | 87 | // http://www.morethanseven.net/2014/04/25/consul/ 88 | 89 | [source,yaml] 90 | .salt/top.sls 91 | ---- 92 | base: 93 | ... 94 | include::../../master/srv/salt/top.sls[tags=dnsmasq] 95 | ---- 96 | 97 | [source,yaml] 98 | .salt/dnsmasq/init.sls 99 | ---- 100 | include::../../master/srv/salt/dnsmasq/init.sls[] 101 | ---- 102 | 103 | [source] 104 | .salt/dnsmasq/networkmanager-dnsmasq.conf 105 | ---- 106 | include::../../master/srv/salt/dnsmasq/conf/networkmanager-dnsmasq.conf[] 107 | ---- 108 | 109 | [source] 110 | .salt/dnsmasq/dnsmasq-consul.conf 111 | ---- 112 | include::../../master/srv/salt/dnsmasq/conf/dnsmasq-consul.conf[] 113 | ---- 114 | 115 | After you've run this successfully, the following lookup will work: 116 | 117 | ---- 118 | $ nslookup web.service.consul 119 | Server: 127.0.0.1 120 | Address: 127.0.0.1#53 121 | 122 | Name: web.service.consul 123 | Address: 192.168.23.21 124 | Name: web.service.consul 125 | Address: 192.168.23.22 126 | Name: web.service.consul 127 | Address: 192.168.23.23 128 | ---- 129 | 130 | -------------------------------------------------------------------------------- /tutorial/50_create_loadbalancer/index.adoc: -------------------------------------------------------------------------------- 1 | ## Creating the load balancer 2 | 3 | ### Install consul-template 4 | 5 | For our load balancer will need a configuration file that is written dynamically with the latest information available for our web nodes. 6 | This job is done by https://github.com/hashicorp/consul-template[consul-template^]. 7 | 8 | This consists of a go binary, a configuration file for `systemd` and a configuration file for consul-template itself. 9 | 10 | To install this, there is the following state: 11 | 12 | [source,yaml] 13 | .top.sls 14 | ---- 15 | base: 16 | ... 17 | include::../../master/srv/salt/top.sls[tags=consultemplate] 18 | ---- 19 | 20 | [source,yaml] 21 | .consul-template/init.sls 22 | ---- 23 | include::../../master/srv/salt/consul-template/init.sls[] 24 | ---- 25 | 26 | Together with its configuration files: 27 | 28 | [source] 29 | .salt/consul-template/conf/consul-template.service 30 | ---- 31 | include::../../master/srv/salt/consul-template/conf/consul-template.service[] 32 | ---- 33 | 34 | [source] 35 | .salt/consul-template/templates/template.conf 36 | ---- 37 | include::../../master/srv/salt/consul-template/templates/template.conf[] 38 | ---- 39 | 40 | Now you can start the new load balancer node and provision it with these two commands: 41 | 42 | ---- 43 | salt-cloud -l debug -p centos-digitalocean lb1 44 | ---- 45 | 46 | Use the following command to have a look where this service is running: 47 | 48 | ---- 49 | salt '*' service.status consul-template 50 | ---- 51 | 52 | In the next step we'll use this service to keep the configuration file of our load balancer up-to-date. 53 | 54 | ### Install nginx 55 | 56 | Now we have multiple web servers running. Now we need to place a load balancer in front of them. The software we can use as a load balancer is nginx (pronounced "engine-X"). 57 | 58 | The installation is straight-forward like installing a web server. First create a state: 59 | 60 | [source,yaml] 61 | .salt/lb/init.sls 62 | ---- 63 | include::../../master/srv/salt/lb/init.sls[] 64 | ---- 65 | 66 | With the following referenced files: 67 | 68 | [source,plain] 69 | .salt/lb/conf/nginx.conf 70 | ---- 71 | include::../../master/srv/salt/lb/conf/nginx.conf[] 72 | ---- 73 | 74 | [source,json] 75 | .salt/lb/conf/nginx.json 76 | ---- 77 | include::../../master/srv/salt/lb/conf/nginx.json[] 78 | ---- 79 | 80 | [source,plain] 81 | .salt/lb/conf/nginx-consul.conf 82 | ---- 83 | include::../../master/srv/salt/lb/conf/nginx-consul.conf[] 84 | ---- 85 | 86 | [source,plain] 87 | .salt/lb/conf/upstream.ctmpl 88 | ---- 89 | include::../../master/srv/salt/lb/conf/upstream.ctmpl[] 90 | ---- 91 | 92 | [source,plain] 93 | .salt/lb/conf/nginx-consul.conf 94 | ---- 95 | include::../../master/srv/salt/lb/conf/nginx-consul.conf[] 96 | ---- 97 | 98 | Then associate this state with a load balancer node: 99 | 100 | [source,yaml] 101 | .top.sls 102 | ---- 103 | base: 104 | ... 105 | 'lb*': 106 | include::../../master/srv/salt/top.sls[tags=lbsetup] 107 | ---- 108 | 109 | Publish this configuration to your nodes: 110 | 111 | ---- 112 | salt '*' state.highstate 113 | ---- 114 | 115 | Point your browser to `http://:9090/` to see the website loadbalancing over multiple servers. 116 | Every time you reload your browser, nginx will direct the request to a different server and you'll 117 | see a different server's IP address. -------------------------------------------------------------------------------- /tutorial/60_cleanup/index.adoc: -------------------------------------------------------------------------------- 1 | ## Cleanup tutorial 2 | 3 | ### Shutdown nodes created by Salt Cloud 4 | 5 | Log in to the salt master. 6 | 7 | Issue the following commands: 8 | 9 | ---- 10 | # query cloud nodes 11 | salt-cloud -Q 12 | # delete nodes 13 | salt-cloud -d web1 web2 web3 web4 web5 lb1 14 | ---- 15 | 16 | Then log off from the master. 17 | 18 | ### Destroy node created with Terraform 19 | 20 | Issue the following command: 21 | 22 | ---- 23 | terraform destroy 24 | ---- 25 | 26 | ### Double Check Digital Ocean Console 27 | 28 | Log in on the web console of DigitalOcean and check if all nodes have been destroyed. 29 | 30 | If not, destroy them manually (as they would otherwise waste money) -------------------------------------------------------------------------------- /tutorial/Gemfile: -------------------------------------------------------------------------------- 1 | # this is used by Travis CI to build the online manual 2 | source 'https://rubygems.org' 3 | gem 'asciidoctor' 4 | gem 'coderay' 5 | gem 'asciidoctor-diagram' 6 | -------------------------------------------------------------------------------- /tutorial/build.bat: -------------------------------------------------------------------------------- 1 | call asciidoctor -r asciidoctor-diagram tutorial.adoc -d book -D output 2 | mkdir output\tutorial 3 | copy 20_getting_started\*.png output\tutorial 4 | copy 45_setup_monitoring\*.png output\tutorial 5 | move tutorial\*.png output\tutorial 6 | move tutorial\*.png.cache output\tutorial 7 | -------------------------------------------------------------------------------- /tutorial/build.sh: -------------------------------------------------------------------------------- 1 | set -x 2 | set -e 3 | mkdir -p $HOME/pages/tutorial 4 | BASEDIR=$(dirname $0) 5 | cd $BASEDIR 6 | cp */*.png $HOME/pages/tutorial 7 | asciidoctor -r asciidoctor-diagram tutorial.adoc -d book -D $HOME/pages 8 | mv */*.png $HOME/pages/tutorial -------------------------------------------------------------------------------- /tutorial/setup.bat: -------------------------------------------------------------------------------- 1 | gem install bundler 2 | bundler 3 | -------------------------------------------------------------------------------- /tutorial/tutorial.adoc: -------------------------------------------------------------------------------- 1 | :toc: left 2 | :linkcss: 3 | :stylesdir: css/ 4 | :source-highlighter: coderay 5 | :numbered: 6 | :icons: font 7 | // using experimental to support btn: macro 8 | :experimental: 9 | :imagesdir: tutorial 10 | 11 | [[tutorial]] 12 | # Manage the cloud with Terraform, Salt, Consul and DigitalOcean 13 | 14 | include::10_introduction/index.adoc[] 15 | 16 | include::20_getting_started/index.adoc[] 17 | 18 | include::30_setup_saltstack/index.adoc[] 19 | 20 | include::35_setup_saltcloud/index.adoc[] 21 | 22 | include::40_create_webservers/index.adoc[] 23 | 24 | include::45_setup_monitoring/index.adoc[] 25 | 26 | include::50_create_loadbalancer/index.adoc[] 27 | 28 | include::60_cleanup/index.adoc[] 29 | 30 | [appendix] 31 | ## License 32 | 33 | Copyright 2015 Alexander Schwartz 34 | 35 | Licensed under the Apache License, Version 2.0 (the "License"); 36 | you may not use this file except in compliance with the License. 37 | You may obtain a copy of the License at 38 | 39 | http://www.apache.org/licenses/LICENSE-2.0 40 | 41 | Unless required by applicable law or agreed to in writing, software 42 | distributed under the License is distributed on an "AS IS" BASIS, 43 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 44 | See the License for the specific language governing permissions and 45 | limitations under the License. 46 | -------------------------------------------------------------------------------- /tutorial/update-gh-pages.sh: -------------------------------------------------------------------------------- 1 | # Taken from and modified http://sleepycoders.blogspot.de/2013/03/sharing-travis-ci-generated-files.html 2 | if [ "$TRAVIS_PULL_REQUEST" == "false" ]; then 3 | echo -e "Starting to update gh-pages\n" 4 | 5 | #go to home and setup git 6 | cd $HOME 7 | git config --global user.email "alexander.schwartz@gmx.net" 8 | git config --global user.name "Travis" 9 | 10 | #using token clone gh-pages branch 11 | git clone --quiet --branch=gh-pages https://${GH_TOKEN}@github.com/ahus1/saltconsul-examples.git gh-pages > /dev/null 12 | 13 | #go into diractory and copy data we're interested in to that directory 14 | cd gh-pages 15 | cp -Rf $HOME/pages/* . 16 | 17 | #add, commit and push files 18 | git add -f . 19 | git commit -m "Travis build $TRAVIS_BUILD_NUMBER pushed to gh-pages" 20 | git push -fq origin gh-pages > /dev/null 21 | 22 | echo -e "Done publishing to gh-pages.\n" 23 | fi 24 | --------------------------------------------------------------------------------