├── .codeclimate.yml ├── .rspec ├── .rubocop.yml ├── .travis.yml ├── CHANGELOG.md ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── Rakefile ├── attributes ├── default.rb ├── master.rb ├── minion.rb ├── proxy.rb └── registry.rb ├── chefignore ├── libraries └── helpers.rb ├── metadata.rb ├── recipes ├── default.rb ├── docker.rb ├── etcd.rb ├── flanneld.rb ├── kube-apiserver.rb ├── kube-controller.rb ├── kube-proxy.rb ├── kube-scheduler.rb ├── kubelet.rb ├── kubernetes.rb ├── master.rb ├── minion.rb ├── network.rb ├── podmaster.rb ├── proxy.rb └── registry.rb ├── templates └── default │ ├── cockpit.service.erb │ ├── docker-env.erb │ ├── docker-registry.erb │ ├── docker.erb │ ├── dockerconfig.erb │ ├── etcd-etcd.erb │ ├── flannel-flanneld.erb │ ├── flannel-network.erb │ ├── kube-apiserver-etcd.erb │ ├── kube-apiserver.erb │ ├── kube-config.erb │ ├── kube-controller-manager.erb │ ├── kube-kubelet-kube-config.erb │ ├── kube-kubelet.erb │ ├── kube-proxy.erb │ ├── kube-scheduler.erb │ ├── podmaster.yaml.erb │ └── proxy.erb └── test ├── integration └── helpers │ └── serverspec │ └── spec_helper.rb └── spec ├── recipes ├── default_spec.rb ├── docker_spec.rb ├── etcd_spec.rb ├── flanneld_spec.rb ├── kube_apiserver_spec.rb ├── kube_controller_spec.rb ├── kube_proxy_spec.rb ├── kube_scheduler_spec.rb ├── kubelet_spec.rb ├── kubernetes_spec.rb ├── master_spec.rb ├── minion_spec.rb ├── network_spec.rb ├── podmaster_spec.rb ├── proxy_spec.rb └── registry_spec.rb └── spec_helper.rb /.codeclimate.yml: -------------------------------------------------------------------------------- 1 | --- 2 | engines: 3 | foodcritic: 4 | enabled: true 5 | rubocop: 6 | enabled: true 7 | ratings: 8 | paths: 9 | - "**.rb" 10 | -------------------------------------------------------------------------------- /.rspec: -------------------------------------------------------------------------------- 1 | --default-path test/spec 2 | --color 3 | -------------------------------------------------------------------------------- /.rubocop.yml: -------------------------------------------------------------------------------- 1 | --- 2 | AllCops: 3 | Exclude: 4 | - 'Guardfile' 5 | - 'Rakefile' 6 | - 'Vagrantfile' 7 | - 'Policyfile.rb' 8 | - 'Berksfile' 9 | - 'Thorfile' 10 | - 'Gemfile' 11 | - 'metadata.rb' 12 | - 'test/**/*' 13 | - 'bin/**' 14 | - 'vendor/**/*' 15 | AlignParameters: 16 | Enabled: false 17 | ClassLength: 18 | Enabled: false 19 | CyclomaticComplexity: 20 | Enabled: false 21 | Documentation: 22 | Enabled: false 23 | Encoding: 24 | Enabled: false 25 | Style/FileName: 26 | Enabled: false 27 | LineLength: 28 | Enabled: false 29 | MethodLength: 30 | Enabled: false 31 | Metrics/AbcSize: 32 | Enabled: false 33 | PerceivedComplexity: 34 | Enabled: false 35 | Style/ClassAndModuleChildren: 36 | Enabled: false 37 | Style/FileName: 38 | Enabled: false 39 | Style/GuardClause: 40 | Enabled: false 41 | Style/PercentLiteralDelimiters: 42 | Enabled: false 43 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | --- 2 | language: ruby 3 | sudo: false 4 | notifications: 5 | slack: bloomberg-rnd:BvYmxrV9xj902XWTRNrkLNkR 6 | script: bundle exec rake travis 7 | rvm: 8 | - 2.2.5 9 | branches: 10 | only: 11 | - master 12 | matrix: 13 | fast_finish: true 14 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Change Log 2 | All notable changes to this project will be documented in this file. 3 | This project will adhere to strict 4 | [Semantic Versioning](http://semver.org/) starting at v1.0.0. 5 | 6 | ## 1.1.0 7 | - Able to configure multiple registries in docker config 8 | 9 | ## 1.0.7 10 | - Fix bug with insecure podmaster connecting to wrong port 11 | 12 | ## 1.0.6 13 | - Make docker secrets and ssl certificate writeout as sensitive to prevent stdout scraping of credentials 14 | 15 | ## 1.0.5 16 | - Add ability to set docker registry credentials for pullling from secured registry 17 | 18 | ## 1.0.4 19 | - Bugfix with proxy in insecure mode 20 | - Always create kube-services group 21 | - Added default attribute for kubelet log lefel 22 | 23 | ## 1.0.3 24 | - Bugfix with kube-proxy deployment 25 | - Fully tested in both secure and insecure mode for HA setup using Kubernetes 1.0.3 and Docker 1.8.2 26 | - Fixed pathing in mount points and templates for podmaster/controller/scheduler 27 | 28 | ## 1.0.0 29 | - Optional deployment of podmaster for master election of master services 30 | - Deployment of scheduler and controller-manager as pods to take advantage of podmaster election 31 | - Selection of podmaster, scheduler, and controller-manager container image source 32 | - Most likely not backwards compatible 33 | - Made library use TLS certificates for etcdctl if applicable 34 | 35 | ## 0.7.1 36 | - Bugfixes related to kube client config 37 | - Elimination of race condition caused by docker0 interface slow startup 38 | 39 | ## 0.7 40 | - Initial public release of the cookbook. 41 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | ## Contributing 2 | 3 | 1. Fork the repository on Github 4 | 2. Create a named feature branch (i.e. `add-new-recipe`) 5 | 3. Write your change 6 | 4. Write tests for your change (if applicable) 7 | 5. Run the tests, ensuring they all pass 8 | 6. Submit a Pull Request 9 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2015-2016, Bloommberg Finance L.P. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # kubernetes-cluster cookbook 2 | [![Build Status](https://img.shields.io/travis/bloomberg/kubernetes-cluster-cookbook.svg)](https://travis-ci.org/bloomberg/kubernetes-cluster-cookbook) 3 | [![Code Quality](https://img.shields.io/codeclimate/github/bloomberg/kubernetes-cluster-cookbook.svg)](https://codeclimate.com/github/bloomberg/kubernetes-cluster-cookbook) 4 | [![Cookbook Version](https://img.shields.io/cookbook/v/kubernetes-cluster.svg)](https://supermarket.chef.io/cookbooks/kubernetes-cluster) 5 | [![License](https://img.shields.io/badge/license-Apache_2-blue.svg)](https://www.apache.org/licenses/LICENSE-2.0) 6 | 7 | Application cookbook which installs and configures a Kubernetes cluster. 8 | 9 | ## Supported Platforms 10 | - RHEL 7.1+ (CentOS 7.1+) 11 | 12 | ## Basic Usage 13 | 14 | [See detailed step by step instructions in the wiki here.](https://github.com/bloomberg/kubernetes-cluster-cookbook/wiki/Beginners-Guide) 15 | 16 | Configure masters using kubernetes-cluster::master 17 | Configure minions using kubernetes-cluster::minion 18 | Configure Docker registry using kubernetes-cluster::registry 19 | 20 | The purpose of this cookbook is to install and configure the proper sevices to create application container clusters. This includes etcd, Kubernetes, Flannel, and Docker- specifically aimed at operating on Enterprise Linux platforms. The best method to using this cookbook is to create a wrapper with specific configurations for your project, adding this cookbook as a dependency. This cookbook assumes you have access to a chef server- however, the cookbook will work fine without it if you override node['kubernetes']['etcd']['members'] in attributes/master.rb node['kubernetes']['master']['fqdn'] in attributes/minion.rb in insecure mode. Secure mode requires more configuration. 21 | 22 | - First, converge your masters, The first master converge may fail due to ETCD cluster sync oddities, I am working for a resolution for this. Just re-converge the masters that fail. 23 | - Second, converge your minions. 24 | 25 | Example solo.json for master 26 | ```json 27 | { 28 | "kubernetes": { 29 | "etcd": { 30 | "members": ["master1.example.com", "master2.example.com", "master3.example.com"] 31 | } 32 | }, 33 | "run_list": ["recipe[kubernetes-cluster::master]"] 34 | } 35 | ``` 36 | 37 | Example solo.json for minion 38 | ```json 39 | { 40 | "kubernetes": { 41 | "master": { 42 | "fqdn": ["master1.example.com", "master2.example.com", "master3.example.com"] 43 | } 44 | }, 45 | "run_list": ["recipe[kubernetes-cluster::minion]"] 46 | } 47 | ``` 48 | 49 | ## Advanced Usage 50 | 51 | As well as configuring a simple Kubernetes cluster, this cookbook also allows for far more advanced configurations. These configurations range from changing flannel network layout, to enabling secure communications, and adding additional Docker regestries. Secure mode will configure SSL and TLS connections for all endpoints for etcd and Kubernetes. This is HIGHLY recommended for production-like purposes. This will require large amounts of prep work. You can also set URLs for additional Docker registries for the minions to get container images from- as well as configuring said registry. 52 | 53 | First, set node['kubernetes']['secure']['enabled'] = 'true' and read below: 54 | 55 | I highly recommend you use a tool like CFSSL (CloudFlare SSL) to create your certificates, check out https://www.digitalocean.com/community/tutorials/how-to-secure-your-coreos-cluster-with-tls-ssl-and-firewall-rules and start at "Use CFSSL to Generate Self-Signed Certificates" 56 | 57 | Masters: 58 | - node['kubernetes']['etcd']['peer']['ca'] - Certificate authority for managing authentication for master to master connections (etcd sync) 59 | - node['kubernetes']['etcd']['peer']['cert'] - Certificate the master identifies as for peer connections (Signed by peer CA) 60 | - node['kubernetes']['etcd']['peer']['key'] - Key matching peer certificate 61 | - node['kubernetes']['etcd']['client']['ca'] - Certificate authority for managing authentication for client to master connections (etcdctl/kubectl/kubelet) 62 | - node['kubernetes']['etcd']['client']['cert'] - Certificate the client identifies as for connections (Signed by client CA) 63 | - node['kubernetes']['etcd']['client']['key'] - Key matching client certificate 64 | 65 | Minions: 66 | - node['kubernetes']['etcd']['client']['ca'] - Certificate authority for verifying cert validity for client to master connections 67 | - node['kubernetes']['etcd']['client']['cert'] - Certificate the client identifies as for connections (Signed by client CA) 68 | - node['kubernetes']['etcd']['client']['key'] - Key matching client certificate 69 | 70 | NOTE: Peer and Client CA can be the same. This will allow for far simpler setup. However, you may use a different CA to more closely manage security if desired. 71 | NOTE: Additional exposed attributes contain notes for usage in the appropriate attributes file. 72 | 73 | Example solo.json for master 74 | ```json 75 | { 76 | "kubernetes": { 77 | "etcd": { 78 | "members": ["master1.example.com", "master2.example.com", "master3.example.com"], 79 | "basedir": "/kube/etcd", 80 | "peer": { 81 | "ca": "-----BEGIN CERTIFICATE-----\ndatadata\n-----END CERTIFICATE-----", 82 | "cert": "-----BEGIN CERTIFICATE-----\ndatadata\n-----END CERTIFICATE-----", 83 | "key": "-----BEGIN KEY-----\ndatadata\n-----END KEY-----" 84 | }, 85 | "client": { 86 | "ca": "-----BEGIN CERTIFICATE-----\ndatadata\n-----END CERTIFICATE-----", 87 | "cert": "-----BEGIN CERTIFICATE-----\ndatadata\n-----END CERTIFICATE-----", 88 | "key": "-----BEGIN KEY-----\ndatadata\n-----END KEY-----" 89 | } 90 | }, 91 | "secure": { 92 | "enabled": "true" 93 | }, 94 | "master": { 95 | "podmaster-source": "registry.example.com:5000/podmaster:1.1", 96 | "scheduler-source": "registry.example.com:5000/scheduler:1.0.3", 97 | "controller-manager-source": "registry.example.com:5000/controller-manager:1.0.3" 98 | } 99 | }, 100 | "docker": { 101 | "environment": { 102 | "docker-registry": "registry.example.com:5000", 103 | "registry-insecure": "registry.example.com:5000", 104 | "docker-basedir": "/kube/docker" 105 | } 106 | }, 107 | "run_list": ["recipe[kubernetes-cluster::master]"] 108 | } 109 | ``` 110 | 111 | Example solo.json for minion 112 | ```json 113 | { 114 | "kubernetes": { 115 | "etcd": { 116 | "client": { 117 | "ca": "-----BEGIN CERTIFICATE-----\ndatadata\n-----END CERTIFICATE-----", 118 | "cert": "-----BEGIN CERTIFICATE-----\ndatadata\n-----END CERTIFICATE-----", 119 | "key": "-----BEGIN KEY-----\ndatadata\n-----END KEY-----" 120 | } 121 | }, 122 | "master": { 123 | "fqdn": ["master1.example.com", "master2.example.com", "master3.example.com"] 124 | }, 125 | "secure": { 126 | "enabled": "true" 127 | } 128 | }, 129 | "docker": { 130 | "environment": { 131 | "docker-registry": "registry.example.com:5000", 132 | "registry-insecure": "registry.example.com:5000", 133 | "docker-basedir": "/kube/docker" 134 | } 135 | }, 136 | "kubelet": { 137 | "pause-source": "registry.example.com:5000/pause:base", 138 | "register": "true" 139 | }, 140 | "run_list": ["recipe[kubernetes-cluster::minion]"] 141 | } 142 | ``` 143 | 144 | Example solo.json for registry 145 | ```json 146 | { 147 | "kubernetes": { 148 | "registry": { 149 | "port": "5000", 150 | "workers": "8", 151 | "storage": "/kube/docker-storage/" 152 | } 153 | }, 154 | "run_list": ["recipe[kubernetes-cluster::registry]"] 155 | } 156 | ``` 157 | 158 | ## Testing 159 | This project will use [Test Kitchen][1] to execute the [ChefSpec][2] tests 160 | on a clean virtual machine. By default Test Kitchen will use [Vagrant][3] 161 | and attempt to start a new virtual machine up from a [default Opscode box][4]. 162 | ```shell 163 | bin/kitchen test default-centos-7.1 164 | ``` 165 | However, no meaningful tests are currently written. This will be fixed. 166 | 167 | [1]: https://kitchen.ci 168 | [2]: https://github.com/sethvargo/chefspec 169 | [3]: http://vagrantup.com 170 | [4]: https://github.com/opscode/bento 171 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env rake 2 | 3 | require 'bundler/setup' 4 | require 'rspec/core/rake_task' 5 | require 'rubocop/rake_task' 6 | require 'foodcritic' 7 | 8 | namespace :style do 9 | desc 'Run Ruby style checks' 10 | RuboCop::RakeTask.new(:ruby) 11 | 12 | desc 'Run Chef style checks' 13 | FoodCritic::Rake::LintTask.new(:chef) 14 | end 15 | 16 | desc 'Run all style checks' 17 | task style: ['style:chef', 'style:ruby'] 18 | 19 | desc 'Run ChefSpec unit tests' 20 | RSpec::Core::RakeTask.new(:unit) do |t| 21 | t.pattern = 'test/spec/**{,/*/**}/*_spec.rb' 22 | end 23 | 24 | desc 'Run style & unit tests on Travis' 25 | task travis: %w(style unit) 26 | 27 | # Default 28 | desc 'Run style, unit, and Vagrant-based integration tests' 29 | task default: %w(style unit vagrant) 30 | -------------------------------------------------------------------------------- /attributes/default.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Cookbook: kubernetes-cluster 3 | # License: Apache 2.0 4 | # 5 | # Copyright 2015-2016, Bloomberg Finance L.P. 6 | # 7 | 8 | # Enable security 9 | # This will tell the cookbook to make your cluster more secure 10 | # This enables SSL certificates, enables authn/authz, and disables insecure ports 11 | # this will make the cookbook run far more complicated- and will require a lot of 12 | # pre-work such as manually generating SSL certificates and figuring your own way 13 | default['kubernetes']['secure']['enabled'] = 'false' 14 | default['kubernetes']['secure']['directory'] = '/etc/kubernetes/secrets' 15 | 16 | # Port to host/access Kubernetes API 17 | default['kubernetes']['insecure']['apiport'] = '8080' 18 | default['kubernetes']['secure']['apiport'] = '8443' 19 | 20 | # Set port for Kubelet communication 21 | default['kubelet']['port'] = '10250' 22 | 23 | # Set pause container source in case of network connectivity issues (eg you are behind a firewall) 24 | default['kubelet']['pause-source'] = nil 25 | 26 | # Set hostname for kubelet 27 | default['kubelet']['hostname'] = node['fqdn'] 28 | 29 | default['docker']['environment'].tap do |environment| 30 | # Add custom docker registry- optionally insecure 31 | environment['docker-registry'] = nil 32 | environment['registry-insecure'] = nil 33 | 34 | # Set docker base directory for local storage- make sure this has plenty of space, optimally its own volume 35 | # This directory will be created if it does not exist 36 | environment['docker-basedir'] = nil 37 | 38 | # Set docker daemon proxy settings 39 | environment['proxy'] = nil 40 | environment['no-proxy'] = nil 41 | end 42 | 43 | # Set optional docker registry credentials 44 | default['docker']['secure']['registry'] = nil 45 | default['docker']['secure']['secret'] = nil 46 | default['docker']['secure']['email'] = nil 47 | 48 | # Set kubelet log level- 0 is lowest 49 | default['kubernetes']['log']['level'] = '5' 50 | 51 | # Package versions 52 | default['kubernetes_cluster']['package']['flannel']['version'] = '>= 0.2.0' 53 | default['kubernetes_cluster']['package']['docker']['name'] = 'docker' 54 | default['kubernetes_cluster']['package']['docker']['version'] = '= 1.8.2' 55 | default['kubernetes_cluster']['package']['kubernetes_client']['version'] = '= 1.0.3' 56 | default['kubernetes_cluster']['package']['kubernetes_master']['version'] = '= 1.0.3' 57 | default['kubernetes_cluster']['package']['kubernetes_node']['version'] = '= 1.0.3' 58 | default['kubernetes_cluster']['package']['etcd']['version'] = '>= 2.0.0' 59 | default['kubernetes_cluster']['package']['cockpit']['enabled'] = true 60 | default['kubernetes_cluster']['package']['cockpit']['version'] = '>= 0.71' 61 | default['kubernetes_cluster']['package']['docker_registry']['version'] = '>= 0.9.1' 62 | default['kubernetes_cluster']['package']['bridge_utils']['version'] = '>= 1.5' 63 | default['kubernetes_cluster']['package']['haproxy']['version'] = '>= 1.5.4' 64 | -------------------------------------------------------------------------------- /attributes/master.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Cookbook: kubernetes-cluster 3 | # License: Apache 2.0 4 | # 5 | # Copyright 2015-2016, Bloomberg Finance L.P. 6 | # 7 | 8 | default['kubernetes']['master'].tap do |master| 9 | # Set the flannel network size of the cluster, and netlength of each node 10 | master['flannel-network'] = '1.80.0.0/16' 11 | master['flannel-netlength'] = '24' 12 | 13 | # Set the Kubernetes service network size 14 | master['service-network'] = '1.90.0.0/16' 15 | 16 | # Get the Kubernetes master hostname 17 | master['fqdn'] = node['fqdn'] 18 | 19 | # Set the source for the podmaster controller-manager, and scheduler container images 20 | # Override this in case of network connectivity issues (eg you are behind a firewall and cannot reach gcr.io) 21 | master['podmaster-source'] = 'gcr.io/google_containers/podmaster:1.1' 22 | master['scheduler-source'] = 'gcr.io/google_containers/kube-scheduler:34d0b8f8b31e27937327961528739bc9' 23 | master['controller-manager-source'] = 'gcr.io/google_containers/kube-controller-manager:fda24638d51a48baa13c35337fcd4793' 24 | end 25 | 26 | default['kubernetes']['etcd'].tap do |etcd| 27 | # etcd client and peer communication ports 28 | etcd['clientport'] = '2379' 29 | etcd['peerport'] = '2380' 30 | 31 | # etcd client name for etcd membership 32 | etcd['clientname'] = node['fqdn'] 33 | 34 | # etcd directory for data storage 35 | etcd['basedir'] = '/var/lib/etcd' 36 | 37 | # etcd token for initial cluster creation 38 | etcd['token'] = 'newtoken' 39 | 40 | # List of etcd members 41 | # This defaults to chef server search capabilities in etcd.rb unless run in chef solo mode 42 | # If you override this, set this to ALL masters eg: 43 | # etcd['members'] = ["node1.example.com", "node2.example.com", "node3.example.com"] 44 | etcd['members'] = nil 45 | 46 | # Set the ssl ca/cert/key for ETCD peer (master to master) communication when ['kubernetes']['secure']['enabled'] = 'true' 47 | etcd['peer']['ca'] = nil 48 | etcd['peer']['cert'] = nil 49 | etcd['peer']['key'] = nil 50 | 51 | # Set the ssl ca/cert/key for ETCD client (minion to master) communication when ['kubernetes']['secure']['enabled'] = 'true' 52 | etcd['client']['ca'] = nil 53 | etcd['client']['cert'] = nil 54 | etcd['client']['key'] = nil 55 | end 56 | 57 | # Tell kubelet not to register on master 58 | override['kubelet']['register'] = 'false' 59 | -------------------------------------------------------------------------------- /attributes/minion.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Cookbook: kubernetes-cluster 3 | # License: Apache 2.0 4 | # 5 | # Copyright 2015-2016, Bloomberg Finance L.P. 6 | # 7 | 8 | # Your kubernetes master fqdn OR cluster proxy 9 | # This defaults to chef server search capabilities in kubelet.rb unless run in chef solo mode 10 | # If you override this, set this to your master fqdn or an array containing the names of your masters: 11 | # default['kubernetes']['master']['fqdn'] = ['myclustermaster1.example.com', 'myclustermaster2.example.com', 'myclustermaster3.example.com' ] 12 | # or default['kubernetes']['master']['fqdn'] = "myclustermaster.example.com" 13 | # ONLY statically set the master fqdn as a single master fqdn if you are not doing an HA setup" 14 | default['kubernetes']['master']['fqdn'] = nil 15 | 16 | # Set the ssl cert/key for ETCD client (minion to master) communication when ['kubernetes']['secure']['enabled'] = 'true' 17 | default['kubernetes']['etcd']['client']['ca'] = nil 18 | default['kubernetes']['etcd']['client']['cert'] = nil 19 | default['kubernetes']['etcd']['client']['key'] = nil 20 | 21 | # Tell kubelet to register on minion 22 | default['kubelet']['register'] = 'true' 23 | -------------------------------------------------------------------------------- /attributes/proxy.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Cookbook: kubernetes-cluster 3 | # License: Apache 2.0 4 | # 5 | # Copyright 2015-2016, Bloomberg Finance L.P. 6 | # 7 | 8 | default['proxy'].tap do |proxy| 9 | # Set api servers for HA 10 | proxy['api']['servers'] = nil 11 | end 12 | -------------------------------------------------------------------------------- /attributes/registry.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Cookbook: kubernetes-cluster 3 | # License: Apache 2.0 4 | # 5 | # Copyright 2015-2016, Bloomberg Finance L.P. 6 | # 7 | 8 | default['kubernetes']['registry'].tap do |registry| 9 | # Set port for registry 10 | registry['port'] = '5000' 11 | 12 | # Set number of workers for Gunicorn 13 | registry['workers'] = '8' 14 | 15 | # Set storage location base for images and registry metadata 16 | # Only supports local storage on registry server right now 17 | # Full path where 'images' and 'registry' directories will be created 18 | registry['storage'] = '/var/docker-registry/' 19 | end 20 | -------------------------------------------------------------------------------- /chefignore: -------------------------------------------------------------------------------- 1 | # Put files/directories that should be ignored in this file when uploading 2 | # or sharing to the community site. 3 | # Lines that start with '# ' are comments. 4 | 5 | # OS generated files # 6 | ###################### 7 | .DS_Store 8 | Icon? 9 | nohup.out 10 | ehthumbs.db 11 | Thumbs.db 12 | 13 | # SASS # 14 | ######## 15 | .sass-cache 16 | 17 | # EDITORS # 18 | ########### 19 | \#* 20 | .#* 21 | *~ 22 | *.sw[a-z] 23 | *.bak 24 | REVISION 25 | TAGS* 26 | tmtags 27 | *_flymake.* 28 | *_flymake 29 | *.tmproj 30 | .project 31 | .settings 32 | mkmf.log 33 | 34 | ## COMPILED ## 35 | ############## 36 | a.out 37 | *.o 38 | *.pyc 39 | *.so 40 | *.com 41 | *.class 42 | *.dll 43 | *.exe 44 | */rdoc/ 45 | 46 | # Testing # 47 | ########### 48 | .watchr 49 | .rspec 50 | spec/* 51 | spec/fixtures/* 52 | test/* 53 | features/* 54 | Guardfile 55 | Procfile 56 | 57 | # SCM # 58 | ####### 59 | .git 60 | */.git 61 | .gitignore 62 | .gitmodules 63 | .gitconfig 64 | .gitattributes 65 | .svn 66 | */.bzr/* 67 | */.hg/* 68 | */.svn/* 69 | 70 | # Berkshelf # 71 | ############# 72 | cookbooks/* 73 | tmp 74 | 75 | # Cookbooks # 76 | ############# 77 | CONTRIBUTING 78 | CHANGELOG* 79 | 80 | # Strainer # 81 | ############ 82 | Colanderfile 83 | Strainerfile 84 | .colander 85 | .strainer 86 | 87 | # Vagrant # 88 | ########### 89 | .vagrant 90 | Vagrantfile 91 | 92 | # Travis # 93 | ########## 94 | .travis.yml 95 | -------------------------------------------------------------------------------- /libraries/helpers.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Cookbook: kubernetes-cluster 3 | # License: Apache 2.0 4 | # 5 | # Copyright 2015-2016, Bloomberg Finance L.P. 6 | # 7 | 8 | module KubernetesCookbook 9 | module Helpers 10 | include Chef::Mixin::ShellOut 11 | 12 | def flannel_network? 13 | if File.exist?('/bin/etcdctl') && File.exist?('/etc/kubernetes/secrets/client.srv.crt') 14 | cmd = shell_out!('etcdctl --peers=https://127.0.0.1:2379 --cert-file=/etc/kubernetes/secrets/client.srv.crt --key-file=/etc/kubernetes/secrets/client.srv.key --ca-file=/etc/kubernetes/secrets/client.ca.crt get coreos.com/network/config | sed "/^$/d"', returns: [0, 2, 4]) 15 | cmd.stderr.empty? && (cmd.stdout != /^Error/) 16 | elsif File.exist?('/bin/etcdctl') 17 | cmd = shell_out!('etcdctl get coreos.com/network/config | sed "/^$/d"', returns: [0, 2, 4]) 18 | cmd.stderr.empty? && (cmd.stdout != /^Error/) 19 | else 20 | puts 'etcdctl is not available' 21 | end 22 | end 23 | end 24 | end 25 | 26 | Chef::Resource::Execute.send(:include, KubernetesCookbook::Helpers) 27 | -------------------------------------------------------------------------------- /metadata.rb: -------------------------------------------------------------------------------- 1 | name 'kubernetes-cluster' 2 | maintainer 'Drew Rapenchuk' 3 | maintainer_email 'drapenchuk@bloomberg.net' 4 | license 'Apache 2.0' 5 | description 'Application cookbook for installing and configuring a Kubernetes cluster.' 6 | long_description 'Application cookbook for installing and configuring a Kubernetes cluster.' 7 | version '1.1.0' 8 | 9 | %w(centos redhat).each do |name| 10 | supports name, '~> 7.1' 11 | end 12 | 13 | source_url 'https://github.com/bloomberg/kubernetes-cluster-cookbook' if respond_to?(:source_url) 14 | issues_url 'https://github.com/bloomberg/kubernetes-cluster-cookbook/issues' if respond_to?(:issues_url) 15 | -------------------------------------------------------------------------------- /recipes/default.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Cookbook: kubernetes-cluster 3 | # License: Apache 2.0 4 | # 5 | # Copyright 2015-2016, Bloomberg Finance L.P. 6 | # 7 | 8 | case node['platform'] 9 | when 'redhat', 'centos', 'fedora' 10 | yum_package "flannel #{node['kubernetes_cluster']['package']['flannel']['version']}" 11 | yum_package "#{node['kubernetes_cluster']['package']['docker']['name']} #{node['kubernetes_cluster']['package']['docker']['version']}" 12 | yum_package "kubernetes-node #{node['kubernetes_cluster']['package']['kubernetes_node']['version']}" 13 | yum_package "bridge-utils #{node['kubernetes_cluster']['package']['bridge_utils']['version']}" 14 | service 'firewalld' do 15 | action [:disable, :stop] 16 | end 17 | end 18 | 19 | group 'kube-services' 20 | 21 | directory node['kubernetes']['secure']['directory'] do 22 | only_if { node['kubernetes']['secure']['enabled'] == 'true' } 23 | owner 'root' 24 | group 'kube-services' 25 | mode '0770' 26 | recursive true 27 | action :create 28 | end 29 | -------------------------------------------------------------------------------- /recipes/docker.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Cookbook: kubernetes-cluster 3 | # License: Apache 2.0 4 | # 5 | # Copyright 2015-2016, Bloomberg Finance L.P. 6 | # 7 | 8 | service 'docker' do 9 | action :enable 10 | end 11 | 12 | execute 'docker-reload' do 13 | command 'systemctl daemon-reload' 14 | action :nothing 15 | end 16 | directory '/etc/systemd/system/docker.service.d' 17 | 18 | if node['docker']['secure']['registry'] 19 | directory '/root/.docker' 20 | template '/root/.docker/config.json' do 21 | source 'dockerconfig.erb' 22 | variables( 23 | docker_registry: node['docker']['secure']['registry'], 24 | docker_secret: node['docker']['secure']['secret'], 25 | docker_email: node['docker']['secure']['email'] 26 | ) 27 | notifies :restart, 'service[docker]', :delayed 28 | sensitive true 29 | end 30 | end 31 | 32 | template '/etc/systemd/system/docker.service.d/http-proxy.conf' do 33 | source 'docker-env.erb' 34 | variables( 35 | docker_proxy: node['docker']['environment']['proxy'], 36 | docker_noproxy: node['docker']['environment']['no-proxy'] 37 | ) 38 | notifies :run, 'execute[docker-reload]', :immediately 39 | end 40 | 41 | template '/etc/sysconfig/docker' do 42 | mode '0640' 43 | source 'docker.erb' 44 | variables( 45 | docker_basedir: node['docker']['environment']['docker-basedir'], 46 | docker_registry: node['docker']['environment']['docker-registry'], 47 | docker_insecureregistry: node['docker']['environment']['registry-insecure'] 48 | ) 49 | notifies :restart, 'service[docker]', :delayed 50 | end 51 | -------------------------------------------------------------------------------- /recipes/etcd.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Cookbook: kubernetes-cluster 3 | # License: Apache 2.0 4 | # 5 | # Copyright 2015-2016, Bloomberg Finance L.P. 6 | # 7 | 8 | unless Chef::Config[:solo] 9 | etcdservers = [] 10 | search(:node, 'tags:"kubernetes.master"') do |s| 11 | etcdservers << s[:fqdn] 12 | end 13 | node.override['kubernetes']['etcd']['members'] = etcdservers 14 | end 15 | 16 | service 'etcd' do 17 | action :enable 18 | end 19 | 20 | directory node['kubernetes']['etcd']['basedir'] do 21 | owner 'etcd' 22 | group 'etcd' 23 | recursive true 24 | end 25 | 26 | template '/etc/etcd/etcd.conf' do 27 | mode '0640' 28 | source 'etcd-etcd.erb' 29 | variables( 30 | etcd_client_name: node['kubernetes']['etcd']['clientname'], 31 | etcd_base_dir: node['kubernetes']['etcd']['basedir'], 32 | etcd_client_token: node['kubernetes']['etcd']['token'], 33 | etcd_client_port: node['kubernetes']['etcd']['clientport'], 34 | etcd_peer_port: node['kubernetes']['etcd']['peerport'], 35 | etcd_members: node['kubernetes']['etcd']['members'], 36 | etcd_cert_dir: node['kubernetes']['secure']['directory'] 37 | ) 38 | notifies :restart, 'service[etcd]', :immediately 39 | end 40 | -------------------------------------------------------------------------------- /recipes/flanneld.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Cookbook: kubernetes-cluster 3 | # License: Apache 2.0 4 | # 5 | # Copyright 2015-2016, Bloomberg Finance L.P. 6 | # 7 | 8 | service 'flanneld' do 9 | action :enable 10 | end 11 | 12 | service 'docker' do 13 | action :nothing 14 | end 15 | 16 | service 'kubelet' do 17 | action :nothing 18 | end 19 | 20 | execute 'redo-docker-bridge' do 21 | command 'ifconfig docker0 down; brctl delbr docker0' 22 | action :nothing 23 | notifies :restart, 'service[docker]', :immediately 24 | end 25 | 26 | template '/etc/sysconfig/flanneld' do 27 | mode '0640' 28 | source 'flannel-flanneld.erb' 29 | variables( 30 | etcd_client_port: node['kubernetes']['etcd']['clientport'], 31 | etcd_cert_dir: node['kubernetes']['secure']['directory'] 32 | ) 33 | notifies :restart, 'service[flanneld]', :immediately 34 | notifies :run, 'execute[redo-docker-bridge]', :delayed 35 | notifies :restart, 'service[kubelet]', :delayed 36 | end 37 | -------------------------------------------------------------------------------- /recipes/kube-apiserver.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Cookbook: kubernetes-cluster 3 | # License: Apache 2.0 4 | # 5 | # Copyright 2015-2016, Bloomberg Finance L.P. 6 | # 7 | 8 | service 'kube-apiserver' do 9 | action :enable 10 | end 11 | 12 | node.default['kubernetes']['master']['fqdn'] = node['fqdn'] 13 | 14 | template '/etc/kubernetes/etcd.client.conf' do 15 | mode '0644' 16 | source 'kube-apiserver-etcd.erb' 17 | variables( 18 | etcd_cert_dir: node['kubernetes']['secure']['directory'], 19 | kubernetes_master: node['kubernetes']['master']['fqdn'], 20 | etcd_client_port: node['kubernetes']['etcd']['clientport'] 21 | ) 22 | only_if { node['kubernetes']['secure']['enabled'] == 'true' } 23 | end 24 | 25 | template '/etc/kubernetes/apiserver' do 26 | mode '0640' 27 | source 'kube-apiserver.erb' 28 | variables( 29 | etcd_client_port: node['kubernetes']['etcd']['clientport'], 30 | kubernetes_api_port: node['kubernetes']['insecure']['apiport'], 31 | kubernetes_secure_api_port: node['kubernetes']['secure']['apiport'], 32 | kubernetes_master: node['kubernetes']['master']['fqdn'], 33 | kubernetes_network: node['kubernetes']['master']['service-network'], 34 | kubelet_port: node['kubelet']['port'], 35 | etcd_cert_dir: node['kubernetes']['secure']['directory'] 36 | ) 37 | notifies :restart, 'service[kube-apiserver]', :immediately 38 | end 39 | -------------------------------------------------------------------------------- /recipes/kube-controller.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Cookbook: kubernetes-cluster 3 | # License: Apache 2.0 4 | # 5 | # Copyright 2015-2016, Bloomberg Finance L.P. 6 | # 7 | 8 | file '/var/log/kube-controller-manager.log' do 9 | action :touch 10 | end 11 | 12 | template '/etc/kubernetes/inactive-manifests/controller-manager.yaml' do 13 | mode '0640' 14 | source 'kube-controller-manager.erb' 15 | variables( 16 | controller_manager_image: node['kubernetes']['master']['controller-manager-source'], 17 | kubernetes_api_port: node['kubernetes']['insecure']['apiport'], 18 | etcd_cert_dir: node['kubernetes']['secure']['directory'] 19 | ) 20 | end 21 | -------------------------------------------------------------------------------- /recipes/kube-proxy.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Cookbook: kubernetes-cluster 3 | # License: Apache 2.0 4 | # 5 | # Copyright 2015-2016, Bloomberg Finance L.P. 6 | # 7 | 8 | service 'kube-proxy' do 9 | action :enable 10 | end 11 | 12 | template '/etc/kubernetes/proxy' do 13 | mode '0640' 14 | source 'kube-proxy.erb' 15 | variables( 16 | etcd_cert_dir: node['kubernetes']['secure']['directory'] 17 | ) 18 | notifies :restart, 'service[kube-proxy]', :immediately 19 | end 20 | -------------------------------------------------------------------------------- /recipes/kube-scheduler.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Cookbook: kubernetes-cluster 3 | # License: Apache 2.0 4 | # 5 | # Copyright 2015-2016, Bloomberg Finance L.P. 6 | # 7 | 8 | file '/var/log/kube-scheduler.log' do 9 | action :touch 10 | end 11 | 12 | template '/etc/kubernetes/inactive-manifests/scheduler.yaml' do 13 | mode '0640' 14 | source 'kube-scheduler.erb' 15 | variables( 16 | kube_scheduler_image: node['kubernetes']['master']['scheduler-source'], 17 | kubernetes_api_port: node['kubernetes']['insecure']['apiport'], 18 | etcd_cert_dir: node['kubernetes']['secure']['directory'] 19 | ) 20 | end 21 | -------------------------------------------------------------------------------- /recipes/kubelet.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Cookbook: kubernetes-cluster 3 | # License: Apache 2.0 4 | # 5 | # Copyright 2015-2016, Bloomberg Finance L.P. 6 | # 7 | 8 | service 'kubelet' do 9 | action :enable 10 | end 11 | 12 | template "#{node['kubernetes']['secure']['directory']}/kube.config" do 13 | mode '770' 14 | group 'kube-services' 15 | source 'kube-kubelet-kube-config.erb' 16 | variables( 17 | kubernetes_secure_api_port: node['kubernetes']['secure']['apiport'], 18 | etcd_cert_dir: node['kubernetes']['secure']['directory'] 19 | ) 20 | only_if { node['kubernetes']['secure']['enabled'] == 'true' } 21 | end 22 | 23 | template '/etc/kubernetes/kubelet' do 24 | mode '0640' 25 | source 'kube-kubelet.erb' 26 | variables( 27 | kubelet_hostname: node['kubelet']['hostname'], 28 | kubernetes_api_port: node['kubernetes']['insecure']['apiport'], 29 | kubelet_port: node['kubelet']['port'], 30 | pause_container: node['kubelet']['pause-source'], 31 | kubernetes_secure_api_port: node['kubernetes']['secure']['apiport'], 32 | etcd_cert_dir: node['kubernetes']['secure']['directory'], 33 | register_node: node['kubelet']['register'] 34 | ) 35 | notifies :restart, 'service[kubelet]', :immediately 36 | end 37 | -------------------------------------------------------------------------------- /recipes/kubernetes.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Cookbook: kubernetes-cluster 3 | # License: Apache 2.0 4 | # 5 | # Copyright 2015-2016, Bloomberg Finance L.P. 6 | # 7 | 8 | template '/etc/kubernetes/config' do 9 | mode '0640' 10 | source 'kube-config.erb' 11 | variables( 12 | kubernetes_log_level: node['kubernetes']['log']['level'], 13 | kubernetes_master: node['fqdn'], 14 | kubernetes_api_port: node['kubernetes']['insecure']['apiport'], 15 | kubernetes_secure_api_port: node['kubernetes']['secure']['apiport'], 16 | etcd_cert_dir: node['kubernetes']['secure']['directory'] 17 | ) 18 | end 19 | -------------------------------------------------------------------------------- /recipes/master.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Cookbook: kubernetes-cluster 3 | # License: Apache 2.0 4 | # 5 | # Copyright 2015-2016, Bloomberg Finance L.P. 6 | # 7 | 8 | node.tag('kubernetes.master') 9 | 10 | include_recipe 'kubernetes-cluster::default' 11 | 12 | case node['platform'] 13 | when 'redhat', 'centos', 'fedora' 14 | yum_package "cockpit #{node['kubernetes_cluster']['package']['cockpit']['version']}" do 15 | only_if { node['kubernetes_cluster']['package']['cockpit']['enabled'] } 16 | end 17 | yum_package "etcd #{node['kubernetes_cluster']['package']['etcd']['version']}" 18 | yum_package "kubernetes-master #{node['kubernetes_cluster']['package']['kubernetes_master']['version']}" 19 | end 20 | 21 | group 'kube-services' do 22 | members %w(etcd kube) 23 | action :modify 24 | end 25 | 26 | directory '/etc/kubernetes/inactive-manifests' do 27 | owner 'root' 28 | group 'kube-services' 29 | mode '0770' 30 | end 31 | 32 | directory '/etc/kubernetes/manifests' do 33 | owner 'root' 34 | group 'kube-services' 35 | mode '0770' 36 | end 37 | 38 | if node['kubernetes']['secure']['enabled'] == 'true' 39 | file 'kubernetes::master[client.ca.crt]' do 40 | path "#{node['kubernetes']['secure']['directory']}/client.ca.crt" 41 | content node['kubernetes']['etcd']['client']['ca'] 42 | owner 'root' 43 | group 'kube-services' 44 | mode '0770' 45 | sensitive true 46 | end 47 | file "#{node['kubernetes']['secure']['directory']}/client.srv.crt" do 48 | content node['kubernetes']['etcd']['peer']['cert'] 49 | owner 'root' 50 | group 'kube-services' 51 | mode '0770' 52 | sensitive true 53 | end 54 | file "#{node['kubernetes']['secure']['directory']}/client.srv.key" do 55 | content node['kubernetes']['etcd']['peer']['key'] 56 | owner 'root' 57 | group 'kube-services' 58 | mode '0770' 59 | sensitive true 60 | end 61 | file "#{node['kubernetes']['secure']['directory']}/peer.ca.crt" do 62 | content node['kubernetes']['etcd']['peer']['ca'] 63 | owner 'root' 64 | group 'kube-services' 65 | mode '0770' 66 | sensitive true 67 | end 68 | file "#{node['kubernetes']['secure']['directory']}/peer.srv.crt" do 69 | content node['kubernetes']['etcd']['peer']['cert'] 70 | owner 'root' 71 | group 'kube-services' 72 | mode '0770' 73 | sensitive true 74 | end 75 | file "#{node['kubernetes']['secure']['directory']}/peer.srv.key" do 76 | content node['kubernetes']['etcd']['peer']['key'] 77 | owner 'root' 78 | group 'kube-services' 79 | mode '0770' 80 | sensitive true 81 | end 82 | end 83 | 84 | include_recipe 'kubernetes-cluster::etcd' 85 | include_recipe 'kubernetes-cluster::kubernetes' 86 | include_recipe 'kubernetes-cluster::kube-apiserver' 87 | include_recipe 'kubernetes-cluster::network' 88 | include_recipe 'kubernetes-cluster::docker' 89 | include_recipe 'kubernetes-cluster::flanneld' 90 | include_recipe 'kubernetes-cluster::kubelet' 91 | include_recipe 'kubernetes-cluster::kube-controller' 92 | include_recipe 'kubernetes-cluster::kube-scheduler' 93 | include_recipe 'kubernetes-cluster::podmaster' 94 | -------------------------------------------------------------------------------- /recipes/minion.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Cookbook: kubernetes-cluster 3 | # License: Apache 2.0 4 | # 5 | # Copyright 2015-2016, Bloomberg Finance L.P. 6 | # 7 | 8 | node.tag('kubernetes.minion') 9 | node.override['kubelet']['register'] = 'true' 10 | 11 | include_recipe 'kubernetes-cluster::default' 12 | include_recipe 'kubernetes-cluster::proxy' 13 | 14 | if node['kubernetes']['secure']['enabled'] == 'true' 15 | group 'kube-services' do 16 | members ['kube'] 17 | action :modify 18 | end 19 | { 20 | "#{node['kubernetes']['secure']['directory']}/client.ca.crt" => node['kubernetes']['etcd']['client']['ca'], 21 | "#{node['kubernetes']['secure']['directory']}/client.srv.crt" => node['kubernetes']['etcd']['client']['cert'], 22 | "#{node['kubernetes']['secure']['directory']}/client.srv.bundle.crt" => "#{node['kubernetes']['etcd']['client']['cert']}\n#{node['kubernetes']['etcd']['client']['ca']}", 23 | "#{node['kubernetes']['secure']['directory']}/client.srv.key" => node['kubernetes']['etcd']['client']['key'] 24 | }.each do |filepath, contents| 25 | file filepath do 26 | content contents 27 | owner 'root' 28 | group 'kube-services' 29 | mode '0770' 30 | sensitive true 31 | end 32 | end 33 | end 34 | 35 | include_recipe 'kubernetes-cluster::kubernetes' 36 | include_recipe 'kubernetes-cluster::kube-proxy' 37 | include_recipe 'kubernetes-cluster::docker' 38 | include_recipe 'kubernetes-cluster::flanneld' 39 | include_recipe 'kubernetes-cluster::kubelet' 40 | -------------------------------------------------------------------------------- /recipes/network.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Cookbook: kubernetes-cluster 3 | # License: Apache 2.0 4 | # 5 | # Copyright 2015-2016, Bloomberg Finance L.P. 6 | # 7 | 8 | if node['kubernetes']['secure']['enabled'] == 'true' 9 | etcdcmd = "etcdctl --peers=https://127.0.0.1:2379 --cert-file=#{node['kubernetes']['secure']['directory']}/client.srv.crt --key-file=#{node['kubernetes']['secure']['directory']}/client.srv.key --ca-file=#{node['kubernetes']['secure']['directory']}/client.ca.crt" 10 | elsif node['kubernetes']['secure']['enabled'] == 'false' 11 | etcdcmd = 'etcdctl' 12 | end 13 | 14 | execute 'getnetwork' do 15 | command "#{etcdcmd} get coreos.com/network/config | sed '/^$/d' > /etc/sysconfig/flannel-network" 16 | only_if { flannel_network? } 17 | action :nothing 18 | end.run_action(:run) 19 | 20 | execute 'setnetwork' do 21 | command "#{etcdcmd} set coreos.com/network/config < /etc/sysconfig/flannel-network" 22 | action :nothing 23 | end 24 | 25 | template '/etc/sysconfig/flannel-network' do 26 | mode '0640' 27 | source 'flannel-network.erb' 28 | variables( 29 | flannel_network: node['kubernetes']['master']['flannel-network'], 30 | flannel_netlength: node['kubernetes']['master']['flannel-netlength'] 31 | ) 32 | notifies :run, 'execute[setnetwork]', :immediately 33 | end 34 | -------------------------------------------------------------------------------- /recipes/podmaster.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Cookbook: kubernetes-cluster 3 | # License: Apache 2.0 4 | # 5 | # Copyright 2015-2016, Bloomberg Finance L.P. 6 | # 7 | 8 | template '/etc/kubernetes/manifests/podmaster.yaml' do 9 | mode '0640' 10 | source 'podmaster.yaml.erb' 11 | variables( 12 | podmaster_image: node['kubernetes']['master']['podmaster-source'], 13 | etcd_client_port: node['kubernetes']['etcd']['clientport'], 14 | etcd_cert_dir: node['kubernetes']['secure']['directory'] 15 | ) 16 | end 17 | -------------------------------------------------------------------------------- /recipes/proxy.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Cookbook: kubernetes-cluster 3 | # License: Apache 2.0 4 | # 5 | # Copyright 2015-2016, Bloomberg Finance L.P. 6 | # 7 | 8 | node.tag('kubernetes.proxy') 9 | 10 | case node['platform'] 11 | when 'redhat', 'centos', 'fedora' 12 | yum_package "haproxy #{node['kubernetes_cluster']['package']['haproxy']['version']}" 13 | end 14 | 15 | service 'haproxy' do 16 | action :enable 17 | end 18 | 19 | template '/etc/haproxy/haproxy.cfg' do 20 | mode '0644' 21 | source 'proxy.erb' 22 | variables( 23 | kubernetes_api_port: node['kubernetes']['insecure']['apiport'], 24 | api_servers: node['kubernetes']['master']['fqdn'], 25 | etcd_client_port: node['kubernetes']['etcd']['clientport'], 26 | kubernetes_secure_api_port: node['kubernetes']['secure']['apiport'] 27 | ) 28 | notifies :restart, 'service[haproxy]', :immediately 29 | end 30 | -------------------------------------------------------------------------------- /recipes/registry.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Cookbook: kubernetes-cluster 3 | # License: Apache 2.0 4 | # 5 | # Copyright 2015-2016, Bloomberg Finance L.P. 6 | # 7 | 8 | node.tag('docker.registry') 9 | 10 | case node['platform'] 11 | when 'redhat', 'centos', 'fedora' 12 | yum_package "docker-registry #{node['kubernetes_cluster']['package']['docker_registry']['version']}" 13 | yum_package "cockpit #{node['kubernetes_cluster']['package']['cockpit']['version']}" 14 | end 15 | 16 | service 'docker-registry' do 17 | action :enable 18 | end 19 | 20 | template '/etc/sysconfig/docker-registry' do 21 | mode '0640' 22 | source 'docker-registry.erb' 23 | variables( 24 | registry_port: node['kubernetes']['registry']['port'], 25 | registry_workers: node['kubernetes']['registry']['workers'], 26 | registry_storage: node['kubernetes']['registry']['storage'] 27 | ) 28 | notifies :restart, 'service[docker-registry]', :immediately 29 | end 30 | -------------------------------------------------------------------------------- /templates/default/cockpit.service.erb: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Cockpit Web Server 3 | Documentation=man:cockpit-ws(8) 4 | 5 | [Service] 6 | ExecStartPre=/usr/sbin/remotectl certificate --ensure --user=root --group=cockpit-ws 7 | ExecStart=/usr/libexec/cockpit-ws 8 | PermissionsStartOnly=true 9 | User=cockpit-ws 10 | Group=cockpit-ws 11 | -------------------------------------------------------------------------------- /templates/default/docker-env.erb: -------------------------------------------------------------------------------- 1 | [Service] 2 | <% unless node['docker']['environment']['proxy'].nil? -%> 3 | Environment="HTTP_PROXY=<%= @docker_proxy %>" 4 | <% end -%> 5 | <% unless node['docker']['environment']['no-proxy'].nil? -%> 6 | Environment="NO_PROXY=<%= @docker_noproxy %>" 7 | <% end -%> 8 | -------------------------------------------------------------------------------- /templates/default/docker-registry.erb: -------------------------------------------------------------------------------- 1 | # File created by Chef! Any local changes will be overwritten next Chef run! 2 | 3 | REGISTRY_ADDRESS=0.0.0.0 4 | SETTINGS_FLAVOR=local 5 | REGISTRY_PORT=<%= @registry_port %> 6 | GUNICORN_WORKERS=<%= @registry_workers %> 7 | STORAGE_PATH=<%= @registry_storage %> 8 | -------------------------------------------------------------------------------- /templates/default/docker.erb: -------------------------------------------------------------------------------- 1 | # File created by Chef! Any local changes will be overwritten next Chef run! 2 | 3 | OPTIONS='--selinux-enabled<% if @docker_basedir -%> -g <%= @docker_basedir %><% end -%>' 4 | DOCKER_CERT_PATH=/etc/docker 5 | <% if @docker_registry -%>ADD_REGISTRY='<% if @docker_registry.kind_of?(Array) -%><% @docker_registry.each do |name| -%> --add-registry <%= name %><% end -%><% else -%> --add-registry <%= @docker_registry %><% end -%>'<% end %> 6 | <% if @docker_insecureregistry -%>INSECURE_REGISTRY='<% if @docker_insecureregistry.kind_of?(Array) -%><% @docker_insecureregistry.each do |name| -%> --insecure-registry <%= name %><% end -%><% else -%> --insecure-registry <%= @docker_insecureregistry %><% end -%>'<% end %> 7 | DOCKER_TMPDIR=<% if @docker_basedir -%><%= @docker_basedir %><% end -%>/tmp 8 | -------------------------------------------------------------------------------- /templates/default/dockerconfig.erb: -------------------------------------------------------------------------------- 1 | { 2 | "auths": { 3 | "<%= @docker_registry %>": { 4 | "auth": "<%= @docker_secret %>", 5 | "email": "<%= @docker_email %>" 6 | } 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /templates/default/etcd-etcd.erb: -------------------------------------------------------------------------------- 1 | # File created by Chef! Any local changes will be overwritten next Chef run! 2 | 3 | ETCD_NAME="<%= @etcd_client_name %>" 4 | ETCD_DATA_DIR="<%= @etcd_base_dir %>/<%= @etcd_client_name %>.etcd" 5 | ETCD_INITIAL_CLUSTER_STATE="new" 6 | ETCD_INITIAL_CLUSTER_TOKEN="<%= @etcd_client_token %>" 7 | 8 | <% if node['kubernetes']['secure']['enabled'] == 'false' -%> 9 | ETCD_LISTEN_CLIENT_URLS="http://<%= @etcd_client_name %>:<%= @etcd_client_port %>,http://127.0.0.1:<%= @etcd_client_port %>" 10 | ETCD_LISTEN_PEER_URLS="http://<%= @etcd_client_name %>:<%= @etcd_peer_port %>,http://127.0.0.1:<%= @etcd_peer_port %>" 11 | ETCD_ADVERTISE_CLIENT_URLS="http://<%= @etcd_client_name %>:<%= @etcd_client_port %>" 12 | ETCD_INITIAL_CLUSTER="<% @etcd_members.each do |name| %><%= name %>=http://<%= name %>:<%= @etcd_peer_port %>,<% end -%>" 13 | ETCD_INITIAL_ADVERTISE_PEER_URLS="http://<%= @etcd_client_name %>:<%= @etcd_peer_port %>" 14 | <% end -%> 15 | 16 | # Secure ETCD configuration parameters go under here when node['kubernetes']['secure']['enabled'] == 'true' 17 | <% if node['kubernetes']['secure']['enabled'] == 'true' -%> 18 | ETCD_LISTEN_CLIENT_URLS="https://<%= @etcd_client_name %>:<%= @etcd_client_port %>,https://127.0.0.1:<%= @etcd_client_port %>" 19 | ETCD_LISTEN_PEER_URLS="https://<%= @etcd_client_name %>:<%= @etcd_peer_port %>,https://127.0.0.1:<%= @etcd_peer_port %>" 20 | ETCD_INITIAL_CLUSTER="<% @etcd_members.each do |name| %><%= name %>=https://<%= name %>:<%= @etcd_peer_port %>,<% end -%>" 21 | ETCD_ADVERTISE_CLIENT_URLS="https://<%= @etcd_client_name %>:<%= @etcd_client_port %>" 22 | ETCD_INITIAL_ADVERTISE_PEER_URLS="https://<%= @etcd_client_name %>:<%= @etcd_peer_port %>" 23 | ETCD_PEER_CLIENT_CERT_AUTH="true" 24 | ETCD_CLIENT_CERT_AUTH="true" 25 | ETCD_CERT_FILE="<%= @etcd_cert_dir %>/client.srv.crt" 26 | ETCD_KEY_FILE="<%= @etcd_cert_dir %>/client.srv.key" 27 | ETCD_TRUSTED_CA_FILE="<%= @etcd_cert_dir %>/client.ca.crt" 28 | ETCD_PEER_CA_FILE="<%= @etcd_cert_dir %>/peer.ca.crt" 29 | ETCD_PEER_CERT_FILE="<%= @etcd_cert_dir %>/peer.srv.crt" 30 | ETCD_PEER_KEY_FILE="<%= @etcd_cert_dir %>/peer.srv.key" 31 | <% end -%> 32 | -------------------------------------------------------------------------------- /templates/default/flannel-flanneld.erb: -------------------------------------------------------------------------------- 1 | # File created by Chef! Any local changes will be overwritten next Chef run! 2 | 3 | FLANNEL_ETCD_KEY="/coreos.com/network" 4 | 5 | # Secure ETCD configuration parameters go under here when node['kubernetes']['secure']['enabled'] == 'false' 6 | <% if node['kubernetes']['secure']['enabled'] == 'false' -%> 7 | FLANNEL_ETCD="http://127.0.0.1:<%= @etcd_client_port %>" 8 | <% end -%> 9 | 10 | # Secure ETCD configuration parameters go under here when node['kubernetes']['secure']['enabled'] == 'true' 11 | <% if node['kubernetes']['secure']['enabled'] == 'true' -%> 12 | FLANNEL_ETCD="https://127.0.0.1:<%= @etcd_client_port %>" 13 | FLANNEL_OPTIONS="--etcd-certfile=<%= @etcd_cert_dir %>/client.srv.crt --etcd-keyfile=<%= @etcd_cert_dir %>/client.srv.key --etcd-cafile=<%= @etcd_cert_dir %>/client.ca.crt" 14 | <% end -%> 15 | -------------------------------------------------------------------------------- /templates/default/flannel-network.erb: -------------------------------------------------------------------------------- 1 | { 2 | "Network": "<%= @flannel_network %>", 3 | "SubnetLen": <%= @flannel_netlength %>, 4 | "Backend": { 5 | "Type": "vxlan", 6 | "VNI": 1 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /templates/default/kube-apiserver-etcd.erb: -------------------------------------------------------------------------------- 1 | { 2 | "cluster": { 3 | "machines": [ "https://<%= @kubernetes_master %>:<%= @etcd_client_port %>" ] 4 | }, 5 | "config": { 6 | "certFile": "<%= @etcd_cert_dir %>/client.srv.crt", 7 | "keyFile": "<%= @etcd_cert_dir %>/client.srv.key" 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /templates/default/kube-apiserver.erb: -------------------------------------------------------------------------------- 1 | # File created by Chef! Any local changes will be overwritten next Chef run! 2 | 3 | # Insecure kube configuration parameters go under here when node['kubernetes']['secure']['enabled'] == 'false' 4 | <% if node['kubernetes']['secure']['enabled'] == 'false' -%> 5 | KUBE_API_ADDRESS="--insecure-bind-address=0.0.0.0" 6 | KUBE_API_PORT="--insecure-port=<%= @kubernetes_api_port %>" 7 | KUBELET_PORT="--kubelet_port=<%= @kubelet_port %>" 8 | KUBE_ETCD_SERVERS="--etcd_servers=http://<%= @kubernetes_master %>:<%= @etcd_client_port %>" 9 | KUBE_SERVICE_ADDRESSES="--service-cluster-ip-range=<%= @kubernetes_network %>" 10 | KUBE_ADMISSION_CONTROL="--admission_control=NamespaceLifecycle,NamespaceExists,LimitRanger,SecurityContextDeny,ResourceQuota" 11 | <% end -%> 12 | 13 | # Secure kube configuration parameters go under here when node['kubernetes']['secure']['enabled'] == 'true' 14 | <% if node['kubernetes']['secure']['enabled'] == 'true' -%> 15 | KUBE_API_ADDRESS="--bind-address=0.0.0.0 --insecure-bind-address=127.0.0.1 " 16 | KUBE_API_PORT="--secure-port=<%= @kubernetes_secure_api_port %> --insecure-port=<%= @kubernetes_api_port %>" 17 | KUBELET_PORT="--kubelet_port=<%= @kubelet_port %>" 18 | KUBE_ETCD_SERVERS="--etcd-config=/etc/kubernetes/etcd.client.conf" 19 | KUBE_SERVICE_ADDRESSES="--service-cluster-ip-range=<%= @kubernetes_network %>" 20 | KUBE_ADMISSION_CONTROL="--admission_control=NamespaceLifecycle,NamespaceExists,LimitRanger,ResourceQuota --client-ca-file=<%= @etcd_cert_dir %>/client.ca.crt --tls-cert-file=<%= @etcd_cert_dir %>/client.srv.crt --tls-private-key-file=<%= @etcd_cert_dir %>/client.srv.key --kubelet-certificate-authority=<%= @etcd_cert_dir %>/client.ca.crt --kubelet-client-certificate=<%= @etcd_cert_dir %>/client.srv.crt --kubelet-client-key=<%= @etcd_cert_dir %>/client.srv.key" 21 | <% end -%> 22 | -------------------------------------------------------------------------------- /templates/default/kube-config.erb: -------------------------------------------------------------------------------- 1 | # File created by Chef! Any local changes will be overwritten next Chef run! 2 | 3 | KUBE_LOGTOSTDERR="--logtostderr=true" 4 | KUBE_LOG_LEVEL="--v=<%= @kubernetes_log_level %>" 5 | KUBE_ALLOW_PRIV="--allow_privileged=false" 6 | 7 | <% if node['kubernetes']['secure']['enabled'] == 'false' -%> 8 | KUBE_MASTER="--master=http://127.0.0.1:<%= @kubernetes_api_port %>" 9 | <% end -%> 10 | 11 | <% if node['kubernetes']['secure']['enabled'] == 'true' -%> 12 | KUBE_OPTIONS="--client-certificate=<%= @etcd_cert_dir %>/client.srv.crt --client-key=<%= @etcd_cert_dir %>/client.srv.key" 13 | KUBE_MASTER="--master=https://127.0.0.1:<%= @kubernetes_secure_api_port %>" 14 | <% end -%> 15 | -------------------------------------------------------------------------------- /templates/default/kube-controller-manager.erb: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: kube-controller-manager 5 | spec: 6 | containers: 7 | - command: 8 | - /bin/sh 9 | - -c 10 | <% if node['kubernetes']['secure']['enabled'] == 'true' -%> 11 | - /usr/local/bin/kube-controller-manager --kubeconfig=<%= @etcd_cert_dir %>/kube.config --v=2 1>>/var/log/kube-controller-manager.log 2>&1 12 | <% end -%> 13 | <% if node['kubernetes']['secure']['enabled'] == 'false' -%> 14 | - /usr/local/bin/kube-controller-manager --master=127.0.0.1:<%= @kubernetes_api_port %> --v=2 1>>/var/log/kube-controller-manager.log 2>&1 15 | <% end -%> 16 | image: <%= @controller_manager_image %> 17 | livenessProbe: 18 | httpGet: 19 | path: /healthz 20 | port: 10252 21 | initialDelaySeconds: 15 22 | timeoutSeconds: 1 23 | name: kube-controller-manager 24 | volumeMounts: 25 | <% if node['kubernetes']['secure']['enabled'] == 'true' -%> 26 | - mountPath: <%= @etcd_cert_dir %> 27 | name: secrets 28 | readOnly: true 29 | <% end -%> 30 | - mountPath: /var/log/kube-controller-manager.log 31 | name: logfile 32 | hostNetwork: true 33 | volumes: 34 | <% if node['kubernetes']['secure']['enabled'] == 'true' -%> 35 | - hostPath: 36 | path: <%= @etcd_cert_dir %> 37 | name: secrets 38 | <% end -%> 39 | - hostPath: 40 | path: /var/log/kube-controller-manager.log 41 | name: logfile 42 | -------------------------------------------------------------------------------- /templates/default/kube-kubelet-kube-config.erb: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Config 3 | users: 4 | - name: kubelet 5 | user: 6 | client-certificate: <%= @etcd_cert_dir %>/client.srv.crt 7 | client-key: <%= @etcd_cert_dir %>/client.srv.key 8 | clusters: 9 | - name: kube 10 | cluster: 11 | server: https://127.0.0.1:<%= @kubernetes_secure_api_port %> 12 | certificate-authority: <%= @etcd_cert_dir %>/client.ca.crt 13 | contexts: 14 | - context: 15 | cluster: kube 16 | user: kubelet 17 | -------------------------------------------------------------------------------- /templates/default/kube-kubelet.erb: -------------------------------------------------------------------------------- 1 | # File created by Chef! Any local changes will be overwritten next Chef run! 2 | 3 | KUBELET_HOSTNAME="--hostname_override=<%= @kubelet_hostname %>" 4 | 5 | # Secure kubelet configuration parameters go under here when node['kubernetes']['secure']['enabled'] == 'false' 6 | <% if node['kubernetes']['secure']['enabled'] == 'false' -%> 7 | KUBELET_API_SERVER="--api_servers=http://127.0.0.1:<%= @kubernetes_api_port %>" 8 | KUBELET_ARGS="--config=/etc/kubernetes/manifests --register-node=<%= @register_node %> <% if @pause_container -%>--pod_infra_container_image=<%= @pause_container %><% end -%>" 9 | <% end -%> 10 | 11 | # Secure kubelet configuration parameters go under here when node['kubernetes']['secure']['enabled'] == 'true' 12 | <% if node['kubernetes']['secure']['enabled'] == 'true' -%> 13 | KUBELET_API_SERVER="--api_servers=https://127.0.0.1:<%= @kubernetes_secure_api_port %>" 14 | KUBELET_ARGS="--config=/etc/kubernetes/manifests --register-node=<%= @register_node %> <% if @pause_container -%>--pod_infra_container_image=<%= @pause_container %><% end -%> --kubeconfig=<%= @etcd_cert_dir %>/kube.config <% if node['kubelet']['register'] == 'true' -%>--tls-cert-file=<%= @etcd_cert_dir %>/client.srv.bundle.crt --tls-private-key-file=<%= @etcd_cert_dir %>/client.srv.key<% end -%>" 15 | <% end -%> 16 | -------------------------------------------------------------------------------- /templates/default/kube-proxy.erb: -------------------------------------------------------------------------------- 1 | <% if node['kubernetes']['secure']['enabled'] == 'true' -%> 2 | KUBE_MASTER="--kubeconfig=<%= @etcd_cert_dir %>/kube.config" 3 | <% end -%> 4 | -------------------------------------------------------------------------------- /templates/default/kube-scheduler.erb: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: kube-scheduler 5 | spec: 6 | hostNetwork: true 7 | containers: 8 | - name: kube-scheduler 9 | image: <%= @kube_scheduler_image %> 10 | command: 11 | - /bin/sh 12 | - -c 13 | <% if node['kubernetes']['secure']['enabled'] == 'true' -%> 14 | - /usr/local/bin/kube-scheduler --kubeconfig=<%= @etcd_cert_dir %>/kube.config --v=2 1>>/var/log/kube-scheduler.log 2>&1 15 | <% end -%> 16 | <% if node['kubernetes']['secure']['enabled'] == 'false' -%> 17 | - /usr/local/bin/kube-scheduler --master=127.0.0.1:<%= @kubernetes_api_port %> --v=2 1>>/var/log/kube-scheduler.log 2>&1 18 | <% end -%> 19 | livenessProbe: 20 | httpGet: 21 | path: /healthz 22 | port: 10251 23 | initialDelaySeconds: 15 24 | timeoutSeconds: 1 25 | volumeMounts: 26 | - mountPath: /var/log/kube-scheduler.log 27 | name: logfile 28 | <% if node['kubernetes']['secure']['enabled'] == 'true' -%> 29 | - mountPath: <%= @etcd_cert_dir %> 30 | name: secrets 31 | readOnly: true 32 | <% end -%> 33 | volumes: 34 | - hostPath: 35 | path: /var/log/kube-scheduler.log 36 | name: logfile 37 | <% if node['kubernetes']['secure']['enabled'] == 'true' -%> 38 | - hostPath: 39 | path: <%= @etcd_cert_dir %> 40 | name: secrets 41 | <% end -%> 42 | -------------------------------------------------------------------------------- /templates/default/podmaster.yaml.erb: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: scheduler-master 5 | spec: 6 | hostNetwork: true 7 | containers: 8 | - name: scheduler-elector 9 | image: <%= @podmaster_image %> 10 | command: 11 | - /podmaster 12 | <% if node['kubernetes']['secure']['enabled'] == 'true' -%> 13 | - --etcd-config=/etc/kubernetes/etcd.client.conf 14 | <% end -%> 15 | <% if node['kubernetes']['secure']['enabled'] == 'false' -%> 16 | - --etcd-servers=http://127.0.0.1:<%= @etcd_client_port %> 17 | <% end -%> 18 | - --key=scheduler 19 | - --source-file=/inactive-manifests/scheduler.yaml 20 | - --dest-file=/manifests/scheduler.yaml 21 | volumeMounts: 22 | <% if node['kubernetes']['secure']['enabled'] == 'true' -%> 23 | - mountPath: /etc/kubernetes 24 | name: kubernetes 25 | readOnly: true 26 | - mountPath: <%= @etcd_cert_dir %> 27 | name: secrets 28 | readOnly: true 29 | <% end -%> 30 | - mountPath: /inactive-manifests 31 | name: inactive-manifests 32 | readOnly: true 33 | - mountPath: /manifests 34 | name: manifests 35 | - name: controller-manager-elector 36 | image: <%= @podmaster_image %> 37 | command: 38 | - /podmaster 39 | <% if node['kubernetes']['secure']['enabled'] == 'true' -%> 40 | - --etcd-config=/etc/kubernetes/etcd.client.conf 41 | <% end -%> 42 | <% if node['kubernetes']['secure']['enabled'] == 'false' -%> 43 | - --etcd-servers=http://127.0.0.1:<%= @etcd_client_port %> 44 | <% end -%> 45 | - --key=controller 46 | - --source-file=/inactive-manifests/controller-manager.yaml 47 | - --dest-file=/manifests/controller-manager.yaml 48 | terminationMessagePath: /dev/termination-log 49 | volumeMounts: 50 | <% if node['kubernetes']['secure']['enabled'] == 'true' -%> 51 | - mountPath: /etc/kubernetes 52 | name: kubernetes 53 | readOnly: true 54 | - mountPath: <%= @etcd_cert_dir %> 55 | name: secrets 56 | readOnly: true 57 | <% end -%> 58 | - mountPath: /inactive-manifests 59 | name: inactive-manifests 60 | readOnly: true 61 | - mountPath: /manifests 62 | name: manifests 63 | volumes: 64 | <% if node['kubernetes']['secure']['enabled'] == 'true' -%> 65 | - hostPath: 66 | path: /etc/kubernetes 67 | name: kubernetes 68 | - hostPath: 69 | path: <%= @etcd_cert_dir %> 70 | name: secrets 71 | <% end -%> 72 | - hostPath: 73 | path: /etc/kubernetes/inactive-manifests 74 | name: inactive-manifests 75 | - hostPath: 76 | path: /etc/kubernetes/manifests 77 | name: manifests 78 | -------------------------------------------------------------------------------- /templates/default/proxy.erb: -------------------------------------------------------------------------------- 1 | #Contents generated by chef! Local changes will be overwritten next converge! 2 | 3 | global 4 | maxconn 4096 5 | log 127.0.0.1 local0 6 | spread-checks 5 7 | daemon 8 | user nobody 9 | group nobody 10 | stats socket /var/lib/haproxy/stats user root group root mode 700 level admin 11 | 12 | defaults 13 | log global 14 | mode http 15 | balance leastconn 16 | maxconn 3072 17 | option httplog 18 | option dontlognull 19 | option abortonclose 20 | option httpclose 21 | retries 3 22 | option redispatch 23 | timeout client 1m 24 | timeout connect 10s 25 | timeout server 1m 26 | timeout check 10s 27 | <% if node['kubernetes']['secure']['enabled'] == 'true' -%> 28 | 29 | frontend api_frontend 30 | bind 127.0.0.1:<%= @kubernetes_secure_api_port %> 31 | option tcplog 32 | mode tcp 33 | default_backend api_backend 34 | 35 | frontend etcd_frontend 36 | bind 127.0.0.1:<%= @etcd_client_port %> 37 | option tcplog 38 | mode tcp 39 | default_backend etcd_backend 40 | 41 | backend api_backend 42 | balance roundrobin 43 | option ssl-hello-chk 44 | mode tcp 45 | <% @api_servers.each do |name| %> 46 | server <%= name %> <%= name %>:<%= @kubernetes_secure_api_port %> check 47 | <% end %> 48 | 49 | backend etcd_backend 50 | balance roundrobin 51 | option ssl-hello-chk 52 | mode tcp 53 | <% @api_servers.each do |name| %> 54 | server <%= name %> <%= name %>:<%= @etcd_client_port %> check 55 | <% end %> 56 | <% end -%> 57 | 58 | <% if node['kubernetes']['secure']['enabled'] == 'false' -%> 59 | 60 | frontend api_frontend 61 | bind 127.0.0.1:<%= @kubernetes_api_port %> 62 | option forwardfor 63 | default_backend api_backend 64 | 65 | frontend etcd_frontend 66 | bind 127.0.0.1:<%= @etcd_client_port %> 67 | option forwardfor 68 | default_backend etcd_backend 69 | 70 | backend api_backend 71 | balance roundrobin 72 | <% @api_servers.each do |name| %> 73 | server <%= name %> <%= name %>:<%= @kubernetes_api_port %> 74 | <% end %> 75 | 76 | backend etcd_backend 77 | balance roundrobin 78 | <% @api_servers.each do |name| %> 79 | server <%= name %> <%= name %>:<%= @etcd_client_port %> 80 | <% end %> 81 | <% end -%> 82 | -------------------------------------------------------------------------------- /test/integration/helpers/serverspec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | require 'serverspec' 2 | 3 | set :backend, :exec 4 | -------------------------------------------------------------------------------- /test/spec/recipes/default_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe_recipe 'kubernetes-cluster::default' do 4 | context 'with default node attributes' do 5 | let(:chef_run) { ChefSpec::SoloRunner.new.converge(described_recipe) } 6 | 7 | it { expect(chef_run).to install_yum_package('flannel >= 0.2.0') } 8 | it { expect(chef_run).to install_yum_package('docker = 1.8.2') } 9 | it { expect(chef_run).to install_yum_package('kubernetes-node = 1.0.3') } 10 | it { expect(chef_run).to install_yum_package('bridge-utils >= 1.5') } 11 | it { expect(chef_run).to disable_service('firewalld') } 12 | it { expect(chef_run).to stop_service('firewalld') } 13 | it { expect(chef_run).to create_group('kube-services') } 14 | it { expect(chef_run).to_not create_directory('/etc/kubernetes/secrets') } 15 | end 16 | 17 | context 'with node[\'kubernetes\'][\'secure\'][\'enabled\'] = true' do 18 | let(:chef_run) do 19 | ChefSpec::SoloRunner.new do |node| 20 | node.normal['kubernetes']['secure']['enabled'] = 'true' 21 | end.converge(described_recipe) 22 | end 23 | 24 | it 'should create directory "/etc/kubernetes/secrets"' do 25 | expect(chef_run).to create_directory('/etc/kubernetes/secrets').with( 26 | owner: 'root', 27 | group: 'kube-services', 28 | mode: '0770', 29 | recursive: true 30 | ) 31 | end 32 | end 33 | end 34 | -------------------------------------------------------------------------------- /test/spec/recipes/docker_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe_recipe 'kubernetes-cluster::docker' do 4 | context 'with default node attributes' do 5 | let(:chef_run) { ChefSpec::SoloRunner.new.converge(described_recipe) } 6 | 7 | it { expect(chef_run).to enable_service('docker') } 8 | it { expect(chef_run).to_not run_execute('docker-reload') } 9 | it { expect(chef_run).to create_directory('/etc/systemd/system/docker.service.d') } 10 | 11 | it 'should create template "/etc/systemd/system/docker.service.d/http-proxy.conf"' do 12 | expect(chef_run).to create_template('/etc/systemd/system/docker.service.d/http-proxy.conf').with( 13 | source: 'docker-env.erb', 14 | variables: { 15 | docker_proxy: nil, 16 | docker_noproxy: nil 17 | } 18 | ) 19 | resource = chef_run.template('/etc/systemd/system/docker.service.d/http-proxy.conf') 20 | expect(resource).to notify('execute[docker-reload]').to(:run).immediately 21 | end 22 | 23 | it 'should create template "/etc/sysconfig/docker"' do 24 | expect(chef_run).to create_template('/etc/sysconfig/docker').with( 25 | mode: '0640', 26 | source: 'docker.erb', 27 | variables: { 28 | docker_basedir: nil, 29 | docker_registry: nil, 30 | docker_insecureregistry: nil 31 | } 32 | ) 33 | resource = chef_run.template('/etc/sysconfig/docker') 34 | expect(resource).to notify('service[docker]').to(:restart).delayed 35 | end 36 | end 37 | 38 | context 'with node[\'docker\'][\'secure\'][\'registry\'] = true' do 39 | let(:chef_run) do 40 | ChefSpec::SoloRunner.new do |node| 41 | node.normal['docker']['secure']['email'] = 'email@example.com' 42 | node.normal['docker']['secure']['registry'] = 'registry.example.com:5000' 43 | node.normal['docker']['secure']['secret'] = 'changeme' 44 | end.converge(described_recipe) 45 | end 46 | 47 | it { expect(chef_run).to create_directory('/root/.docker') } 48 | 49 | it 'should create template "/root/.docker/config.json"' do 50 | expect(chef_run).to create_template('/root/.docker/config.json').with( 51 | source: 'dockerconfig.erb', 52 | variables: { 53 | docker_registry: 'registry.example.com:5000', 54 | docker_secret: 'changeme', 55 | docker_email: 'email@example.com' 56 | }, 57 | sensitive: true 58 | ) 59 | resource = chef_run.template('/root/.docker/config.json') 60 | expect(resource).to notify('service[docker]').to(:restart).delayed 61 | end 62 | end 63 | end 64 | -------------------------------------------------------------------------------- /test/spec/recipes/etcd_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe_recipe 'kubernetes-cluster::etcd' do 4 | context 'with default node attributes' do 5 | let(:chef_run) { ChefSpec::SoloRunner.new.converge(described_recipe) } 6 | 7 | it { expect(chef_run).to enable_service('etcd') } 8 | 9 | it 'should create directory "/var/lib/etcd"' do 10 | expect(chef_run).to create_directory('/var/lib/etcd').with( 11 | owner: 'etcd', 12 | group: 'etcd', 13 | recursive: true 14 | ) 15 | end 16 | 17 | it 'should create template "/etc/etcd/etcd.conf"' do 18 | expect(chef_run).to create_template('/etc/etcd/etcd.conf').with( 19 | mode: '0640', 20 | source: 'etcd-etcd.erb', 21 | variables: { 22 | etcd_client_name: 'fauxhai.local', 23 | etcd_base_dir: '/var/lib/etcd', 24 | etcd_client_token: 'newtoken', 25 | etcd_client_port: '2379', 26 | etcd_peer_port: '2380', 27 | etcd_members: nil, 28 | etcd_cert_dir: '/etc/kubernetes/secrets' 29 | } 30 | ) 31 | resource = chef_run.template('/etc/etcd/etcd.conf') 32 | expect(resource).to notify('service[etcd]').to(:restart).immediately 33 | end 34 | end 35 | end 36 | -------------------------------------------------------------------------------- /test/spec/recipes/flanneld_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe_recipe 'kubernetes-cluster::flanneld' do 4 | context 'with default node attributes' do 5 | let(:chef_run) { ChefSpec::SoloRunner.new.converge(described_recipe) } 6 | 7 | it { expect(chef_run).to enable_service('flanneld') } 8 | it { expect(chef_run).to_not enable_service('docker') } 9 | it { expect(chef_run).to_not start_service('docker') } 10 | it { expect(chef_run).to_not enable_service('kubelet') } 11 | it { expect(chef_run).to_not start_service('kubelet') } 12 | it { expect(chef_run).to_not run_execute('redo-docker-bridge') } 13 | 14 | it 'should not run execute "redo-docker-bridge"' do 15 | expect(chef_run).to_not run_execute('redo-docker-bridge').with( 16 | command: 'ifconfig docker0 down; brctl delbr docker0 adsf' 17 | ) 18 | # resource = chef_run.execute('redo-docker-bridg') 19 | # expect(resource).to notify('service[docker]').to(:restart).immediately 20 | end 21 | 22 | it 'should create template "/etc/sysconfig/flanneld"' do 23 | expect(chef_run).to create_template('/etc/sysconfig/flanneld').with( 24 | mode: '0640', 25 | source: 'flannel-flanneld.erb', 26 | variables: { 27 | etcd_client_port: '2379', 28 | etcd_cert_dir: '/etc/kubernetes/secrets' 29 | } 30 | ) 31 | resource = chef_run.template('/etc/sysconfig/flanneld') 32 | expect(resource).to notify('service[flanneld]').to(:restart).immediately 33 | expect(resource).to notify('execute[redo-docker-bridge]').to(:run).delayed 34 | expect(resource).to notify('service[kubelet]').to(:restart).delayed 35 | end 36 | end 37 | end 38 | -------------------------------------------------------------------------------- /test/spec/recipes/kube_apiserver_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe_recipe 'kubernetes-cluster::kube-apiserver' do 4 | context 'with default node attributes' do 5 | let(:chef_run) { ChefSpec::SoloRunner.new.converge(described_recipe) } 6 | 7 | it { expect(chef_run).to enable_service('kube-apiserver') } 8 | it { expect(chef_run.node['kubernetes']['master']['fqdn']).to eq('fauxhai.local') } 9 | it { expect(chef_run).to_not create_template('/etc/kubernetes/etcd.client.conf') } 10 | 11 | it 'should create template "/etc/kubernetes/apiserver"' do 12 | expect(chef_run).to create_template('/etc/kubernetes/apiserver').with( 13 | mode: '0640', 14 | source: 'kube-apiserver.erb', 15 | variables: { 16 | etcd_client_port: '2379', 17 | etcd_cert_dir: '/etc/kubernetes/secrets', 18 | kubelet_port: '10250', 19 | kubernetes_api_port: '8080', 20 | kubernetes_master: 'fauxhai.local', 21 | kubernetes_network: '1.90.0.0/16', 22 | kubernetes_secure_api_port: '8443' 23 | } 24 | ) 25 | resource = chef_run.template('/etc/kubernetes/apiserver') 26 | expect(resource).to notify('service[kube-apiserver]').to(:restart).immediately 27 | end 28 | end 29 | 30 | context 'with node[\'kubernetes\'][\'secure\'][\'enabled\'] = true' do 31 | let(:chef_run) do 32 | ChefSpec::SoloRunner.new do |node| 33 | node.normal['kubernetes']['secure']['enabled'] = 'true' 34 | end.converge(described_recipe) 35 | end 36 | 37 | it 'should create template "/etc/kubernetes/etcd.client.conf"' do 38 | expect(chef_run).to create_template('/etc/kubernetes/etcd.client.conf').with( 39 | mode: '0644', 40 | source: 'kube-apiserver-etcd.erb', 41 | variables: { 42 | etcd_client_port: '2379', 43 | etcd_cert_dir: '/etc/kubernetes/secrets', 44 | kubernetes_master: 'fauxhai.local' 45 | } 46 | ) 47 | end 48 | end 49 | end 50 | -------------------------------------------------------------------------------- /test/spec/recipes/kube_controller_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe_recipe 'kubernetes-cluster::kube-controller' do 4 | context 'with default node attributes' do 5 | let(:chef_run) { ChefSpec::SoloRunner.new.converge(described_recipe) } 6 | 7 | it { expect(chef_run).to touch_file('/var/log/kube-controller-manager.log') } 8 | 9 | it 'should create template "/etc/kubernetes/inactive-manifests/controller-manager.yaml"' do 10 | expect(chef_run).to create_template('/etc/kubernetes/inactive-manifests/controller-manager.yaml').with( 11 | mode: '0640', 12 | source: 'kube-controller-manager.erb', 13 | variables: { 14 | controller_manager_image: 'gcr.io/google_containers/kube-controller-manager:fda24638d51a48baa13c35337fcd4793', 15 | etcd_cert_dir: '/etc/kubernetes/secrets', 16 | kubernetes_api_port: '8080' 17 | } 18 | ) 19 | end 20 | end 21 | end 22 | -------------------------------------------------------------------------------- /test/spec/recipes/kube_proxy_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe_recipe 'kubernetes-cluster::kube-proxy' do 4 | context 'with default node attributes' do 5 | let(:chef_run) { ChefSpec::SoloRunner.new.converge(described_recipe) } 6 | 7 | it { expect(chef_run).to enable_service('kube-proxy') } 8 | 9 | it 'should create template "/etc/kubernetes/proxy"' do 10 | expect(chef_run).to create_template('/etc/kubernetes/proxy').with( 11 | mode: '0640', 12 | source: 'kube-proxy.erb', 13 | variables: { 14 | etcd_cert_dir: '/etc/kubernetes/secrets' 15 | } 16 | ) 17 | resource = chef_run.template('/etc/kubernetes/proxy') 18 | expect(resource).to notify('service[kube-proxy]').to(:restart).immediately 19 | end 20 | end 21 | end 22 | -------------------------------------------------------------------------------- /test/spec/recipes/kube_scheduler_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe_recipe 'kubernetes-cluster::kube-scheduler' do 4 | context 'with default node attributes' do 5 | let(:chef_run) { ChefSpec::SoloRunner.new.converge(described_recipe) } 6 | 7 | it { expect(chef_run).to touch_file('/var/log/kube-scheduler.log') } 8 | 9 | it 'should create template "/etc/kubernetes/inactive-manifests/scheduler.yaml"' do 10 | expect(chef_run).to create_template('/etc/kubernetes/inactive-manifests/scheduler.yaml').with( 11 | mode: '0640', 12 | source: 'kube-scheduler.erb', 13 | variables: { 14 | etcd_cert_dir: '/etc/kubernetes/secrets', 15 | kube_scheduler_image: 'gcr.io/google_containers/kube-scheduler:34d0b8f8b31e27937327961528739bc9', 16 | kubernetes_api_port: '8080' 17 | } 18 | ) 19 | end 20 | end 21 | end 22 | -------------------------------------------------------------------------------- /test/spec/recipes/kubelet_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe_recipe 'kubernetes-cluster::kubelet' do 4 | context 'with default node attributes' do 5 | let(:chef_run) { ChefSpec::SoloRunner.new.converge(described_recipe) } 6 | 7 | it { expect(chef_run).to enable_service('kubelet') } 8 | 9 | it 'should create template "/etc/kubernetes/kubelet"' do 10 | expect(chef_run).to create_template('/etc/kubernetes/kubelet').with( 11 | mode: '0640', 12 | source: 'kube-kubelet.erb', 13 | variables: { 14 | etcd_cert_dir: '/etc/kubernetes/secrets', 15 | kubelet_port: '10250', 16 | kubernetes_api_port: '8080', 17 | kubernetes_secure_api_port: '8443', 18 | kubelet_hostname: 'fauxhai.local', 19 | pause_container: nil, 20 | register_node: "false" 21 | } 22 | ) 23 | resource = chef_run.template('/etc/kubernetes/kubelet') 24 | expect(resource).to notify('service[kubelet]').to(:restart).immediately 25 | end 26 | end 27 | 28 | context 'with node[\'kubernetes\'][\'secure\'][\'enabled\'] = true' do 29 | let(:chef_run) do 30 | ChefSpec::SoloRunner.new do |node| 31 | node.normal['kubernetes']['secure']['enabled'] = 'true' 32 | end.converge(described_recipe) 33 | end 34 | 35 | it 'should create template "/etc/kubernetes/secrets/kube.config"' do 36 | expect(chef_run).to create_template('/etc/kubernetes/secrets/kube.config').with( 37 | mode: '770', 38 | group: 'kube-services', 39 | source: 'kube-kubelet-kube-config.erb', 40 | variables: { 41 | etcd_cert_dir: '/etc/kubernetes/secrets', 42 | kubernetes_secure_api_port: '8443' 43 | } 44 | ) 45 | end 46 | end 47 | end 48 | -------------------------------------------------------------------------------- /test/spec/recipes/kubernetes_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe_recipe 'kubernetes-cluster::kubernetes' do 4 | context 'with default node attributes' do 5 | let(:chef_run) { ChefSpec::SoloRunner.new.converge(described_recipe) } 6 | 7 | it 'should create template "/etc/kubernetes/config"' do 8 | expect(chef_run).to create_template('/etc/kubernetes/config').with( 9 | mode: '0640', 10 | source: 'kube-config.erb', 11 | variables: { 12 | etcd_cert_dir: '/etc/kubernetes/secrets', 13 | kubernetes_api_port: '8080', 14 | kubernetes_log_level: '5', 15 | kubernetes_master: 'fauxhai.local', 16 | kubernetes_secure_api_port: '8443' 17 | } 18 | ) 19 | end 20 | end 21 | end 22 | -------------------------------------------------------------------------------- /test/spec/recipes/master_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe_recipe 'kubernetes-cluster::master' do 4 | before do 5 | global_stubs_include_recipe 6 | end 7 | 8 | context 'with default node attributes' do 9 | let(:chef_run) { ChefSpec::SoloRunner.new.converge(described_recipe) } 10 | 11 | it { expect(chef_run.node.tags).to eq(['kubernetes.master']) } 12 | it { expect(chef_run).to install_yum_package('cockpit >= 0.71') } 13 | it { expect(chef_run).to install_yum_package('etcd >= 2.0.0') } 14 | it { expect(chef_run).to install_yum_package('kubernetes-master = 1.0.3') } 15 | 16 | it 'should include recipe kubernetes-cluster::default' do 17 | allow_any_instance_of(Chef::Recipe).to receive(:include_recipe).with('kubernetes-cluster::default') 18 | chef_run 19 | end 20 | 21 | it 'should create group "kube-services"' do 22 | expect(chef_run).to modify_group('kube-services').with( 23 | members: %w(etcd kube) 24 | ) 25 | end 26 | 27 | it 'should create directory "/etc/kubernetes/inactive-manifests"' do 28 | expect(chef_run).to create_directory('/etc/kubernetes/inactive-manifests').with( 29 | owner: 'root', 30 | group: 'kube-services', 31 | mode: '0770' 32 | ) 33 | end 34 | 35 | it 'should create directory "/etc/kubernetes/manifests"' do 36 | expect(chef_run).to create_directory('/etc/kubernetes/manifests').with( 37 | owner: 'root', 38 | group: 'kube-services', 39 | mode: '0770' 40 | ) 41 | end 42 | 43 | it { expect(chef_run).to_not create_file('/etc/kubernetes/secrets/client.ca.crt') } 44 | it { expect(chef_run).to_not create_file('/etc/kubernetes/secrets/client.srv.crt') } 45 | it { expect(chef_run).to_not create_file('/etc/kubernetes/secrets/client.srv.key') } 46 | it { expect(chef_run).to_not create_file('/etc/kubernetes/secrets/peer.ca.crt') } 47 | it { expect(chef_run).to_not create_file('/etc/kubernetes/secrets/peer.srv.crt') } 48 | it { expect(chef_run).to_not create_file('/etc/kubernetes/secrets/peer.srv.key') } 49 | 50 | it 'should include recipe kubernetes-cluster::etcd' do 51 | allow_any_instance_of(Chef::Recipe).to receive(:include_recipe).with('kubernetes-cluster::etcd') 52 | chef_run 53 | end 54 | 55 | it 'should include recipe kubernetes-cluster::kubernetes' do 56 | allow_any_instance_of(Chef::Recipe).to receive(:include_recipe).with('kubernetes-cluster::kubernetes') 57 | chef_run 58 | end 59 | 60 | it 'should include recipe kubernetes-cluster::kube-apiserver' do 61 | allow_any_instance_of(Chef::Recipe).to receive(:include_recipe).with('kubernetes-cluster::kube-apiserver') 62 | chef_run 63 | end 64 | 65 | it 'should include recipe kubernetes-cluster::network' do 66 | allow_any_instance_of(Chef::Recipe).to receive(:include_recipe).with('kubernetes-cluster::network') 67 | chef_run 68 | end 69 | 70 | it 'should include recipe kubernetes-cluster::docker' do 71 | allow_any_instance_of(Chef::Recipe).to receive(:include_recipe).with('kubernetes-cluster::docker') 72 | chef_run 73 | end 74 | 75 | it 'should include recipe kubernetes-cluster::flanneld' do 76 | allow_any_instance_of(Chef::Recipe).to receive(:include_recipe).with('kubernetes-cluster::flanneld') 77 | chef_run 78 | end 79 | 80 | it 'should include recipe kubernetes-cluster::kubelet' do 81 | allow_any_instance_of(Chef::Recipe).to receive(:include_recipe).with('kubernetes-cluster::kubelet') 82 | chef_run 83 | end 84 | 85 | it 'should include recipe kubernetes-cluster::kube-controller' do 86 | allow_any_instance_of(Chef::Recipe).to receive(:include_recipe).with('kubernetes-cluster::kube-controller') 87 | chef_run 88 | end 89 | 90 | it 'should include recipe kubernetes-cluster::kube-scheduler' do 91 | allow_any_instance_of(Chef::Recipe).to receive(:include_recipe).with('kubernetes-cluster::kube-scheduler') 92 | chef_run 93 | end 94 | 95 | it 'should include recipe kubernetes-cluster::podmaster' do 96 | allow_any_instance_of(Chef::Recipe).to receive(:include_recipe).with('kubernetes-cluster::podmaster') 97 | chef_run 98 | end 99 | end 100 | 101 | context 'with node[\'kubernetes\'][\'secure\'][\'enabled\'] = true' do 102 | let(:chef_run) do 103 | ChefSpec::SoloRunner.new do |node| 104 | node.normal['kubernetes']['secure']['enabled'] = 'true' 105 | end.converge(described_recipe) 106 | end 107 | 108 | it 'should create file "/etc/kubernetes/secrets/client.ca.crt"' do 109 | expect(chef_run).to create_file('/etc/kubernetes/secrets/client.ca.crt').with( 110 | contents: nil, 111 | owner: 'root', 112 | group: 'kube-services', 113 | mode: '0770', 114 | sensitive: true 115 | ) 116 | end 117 | 118 | it 'should create file "/etc/kubernetes/secrets/client.srv.crt"' do 119 | expect(chef_run).to create_file('/etc/kubernetes/secrets/client.srv.crt').with( 120 | contents: nil, 121 | owner: 'root', 122 | group: 'kube-services', 123 | mode: '0770', 124 | sensitive: true 125 | ) 126 | end 127 | 128 | it 'should create file "/etc/kubernetes/secrets/client.srv.key"' do 129 | expect(chef_run).to create_file('/etc/kubernetes/secrets/client.srv.key').with( 130 | contents: nil, 131 | owner: 'root', 132 | group: 'kube-services', 133 | mode: '0770', 134 | sensitive: true 135 | ) 136 | end 137 | 138 | it 'should create file "/etc/kubernetes/secrets/peer.ca.crt"' do 139 | expect(chef_run).to create_file('/etc/kubernetes/secrets/peer.ca.crt').with( 140 | contents: nil, 141 | owner: 'root', 142 | group: 'kube-services', 143 | mode: '0770', 144 | sensitive: true 145 | ) 146 | end 147 | 148 | it 'should create file "/etc/kubernetes/secrets/peer.srv.crt"' do 149 | expect(chef_run).to create_file('/etc/kubernetes/secrets/peer.srv.crt').with( 150 | contents: nil, 151 | owner: 'root', 152 | group: 'kube-services', 153 | mode: '0770', 154 | sensitive: true 155 | ) 156 | end 157 | 158 | it 'should create file "/etc/kubernetes/secrets/peer.srv.key"' do 159 | expect(chef_run).to create_file('/etc/kubernetes/secrets/peer.srv.key').with( 160 | contents: nil, 161 | owner: 'root', 162 | group: 'kube-services', 163 | mode: '0770', 164 | sensitive: true 165 | ) 166 | end 167 | end 168 | end 169 | -------------------------------------------------------------------------------- /test/spec/recipes/minion_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe_recipe 'kubernetes-cluster::minion' do 4 | before do 5 | global_stubs_include_recipe 6 | end 7 | 8 | context 'with default node attributes' do 9 | let(:chef_run) { ChefSpec::SoloRunner.new.converge(described_recipe) } 10 | 11 | it { expect(chef_run.node.tags).to eq(['kubernetes.minion']) } 12 | it { expect(chef_run.node['kubelet']['register']).to eq('true') } 13 | 14 | it 'should include recipe kubernetes-cluster::default' do 15 | allow_any_instance_of(Chef::Recipe).to receive(:include_recipe).with('kubernetes-cluster::default') 16 | chef_run 17 | end 18 | 19 | it 'should include recipe kubernetes-cluster::proxy' do 20 | allow_any_instance_of(Chef::Recipe).to receive(:include_recipe).with('kubernetes-cluster::proxy') 21 | chef_run 22 | end 23 | 24 | it { expect(chef_run).to_not modify_group('kube-services') } 25 | it { expect(chef_run).to_not create_file('/etc/kubernetes/secrets/client.ca.crt') } 26 | it { expect(chef_run).to_not create_file('/etc/kubernetes/secrets/client.srv.crt') } 27 | it { expect(chef_run).to_not create_file('/etc/kubernetes/secrets/client.srv.bundle.crt') } 28 | it { expect(chef_run).to_not create_file('/etc/kubernetes/secrets/client.srv.key') } 29 | 30 | it 'should include recipe kubernetes-cluster::kubernetes' do 31 | allow_any_instance_of(Chef::Recipe).to receive(:include_recipe).with('kubernetes-cluster::kubernetes') 32 | chef_run 33 | end 34 | 35 | it 'should include recipe kubernetes-cluster::kube-proxy' do 36 | allow_any_instance_of(Chef::Recipe).to receive(:include_recipe).with('kubernetes-cluster::kube-proxy') 37 | chef_run 38 | end 39 | 40 | it 'should include recipe kubernetes-cluster::docker' do 41 | allow_any_instance_of(Chef::Recipe).to receive(:include_recipe).with('kubernetes-cluster::docker') 42 | chef_run 43 | end 44 | 45 | it 'should include recipe kubernetes-cluster::flanneld' do 46 | allow_any_instance_of(Chef::Recipe).to receive(:include_recipe).with('kubernetes-cluster::flanneld') 47 | chef_run 48 | end 49 | 50 | it 'should include recipe kubernetes-cluster::kubelet' do 51 | allow_any_instance_of(Chef::Recipe).to receive(:include_recipe).with('kubernetes-cluster::kubelet') 52 | chef_run 53 | end 54 | end 55 | 56 | context 'with node[\'kubernetes\'][\'secure\'][\'enabled\'] = true' do 57 | let(:chef_run) do 58 | ChefSpec::SoloRunner.new do |node| 59 | node.normal['kubernetes']['secure']['enabled'] = 'true' 60 | end.converge(described_recipe) 61 | end 62 | 63 | it 'should create group "kube-services"' do 64 | expect(chef_run).to modify_group('kube-services').with( 65 | members: %w(kube) 66 | ) 67 | end 68 | 69 | it 'should create file "/etc/kubernetes/secrets/client.ca.crt"' do 70 | expect(chef_run).to create_file('/etc/kubernetes/secrets/client.ca.crt').with( 71 | contents: nil, 72 | owner: 'root', 73 | group: 'kube-services', 74 | mode: '0770', 75 | sensitive: true 76 | ) 77 | end 78 | 79 | it 'should create file "/etc/kubernetes/secrets/client.srv.crt"' do 80 | expect(chef_run).to create_file('/etc/kubernetes/secrets/client.srv.crt').with( 81 | contents: nil, 82 | owner: 'root', 83 | group: 'kube-services', 84 | mode: '0770', 85 | sensitive: true 86 | ) 87 | end 88 | 89 | it 'should create file "/etc/kubernetes/secrets/client.srv.key"' do 90 | expect(chef_run).to create_file('/etc/kubernetes/secrets/client.srv.bundle.crt').with( 91 | contents: nil, 92 | owner: 'root', 93 | group: 'kube-services', 94 | mode: '0770', 95 | sensitive: true 96 | ) 97 | end 98 | 99 | it 'should create file "/etc/kubernetes/secrets/client.srv.key"' do 100 | expect(chef_run).to create_file('/etc/kubernetes/secrets/client.srv.key').with( 101 | contents: nil, 102 | owner: 'root', 103 | group: 'kube-services', 104 | mode: '0770', 105 | sensitive: true 106 | ) 107 | end 108 | end 109 | end 110 | -------------------------------------------------------------------------------- /test/spec/recipes/network_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require_relative '../../../libraries/helpers' 3 | 4 | describe_recipe 'kubernetes-cluster::network' do 5 | context 'with default node attributes' do 6 | let(:chef_run) { ChefSpec::SoloRunner.new.converge(described_recipe) } 7 | 8 | it { expect(chef_run).to_not run_execute('getnetwork') } 9 | it { expect(chef_run).to_not run_execute('setnetwork') } 10 | 11 | it 'should create template "/etc/sysconfig/flannel-network"' do 12 | expect(chef_run).to create_template('/etc/sysconfig/flannel-network').with( 13 | mode: '0640', 14 | source: 'flannel-network.erb', 15 | variables: { 16 | flannel_netlength:'24', 17 | flannel_network: '1.80.0.0/16' 18 | } 19 | ) 20 | resource = chef_run.template('/etc/sysconfig/flannel-network') 21 | expect(resource).to notify('execute[setnetwork]').to(:run).immediately 22 | end 23 | end 24 | end 25 | -------------------------------------------------------------------------------- /test/spec/recipes/podmaster_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe_recipe 'kubernetes-cluster::podmaster' do 4 | context 'with default node attributes' do 5 | let(:chef_run) { ChefSpec::SoloRunner.new.converge(described_recipe) } 6 | 7 | it 'should create template "/etc/kubernetes/manifests/podmaster.yaml"' do 8 | expect(chef_run).to create_template('/etc/kubernetes/manifests/podmaster.yaml').with( 9 | mode: '0640', 10 | source: 'podmaster.yaml.erb', 11 | variables: { 12 | etcd_client_port: '2379', 13 | etcd_cert_dir: '/etc/kubernetes/secrets', 14 | podmaster_image: 'gcr.io/google_containers/podmaster:1.1' 15 | } 16 | ) 17 | end 18 | end 19 | end 20 | -------------------------------------------------------------------------------- /test/spec/recipes/proxy_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe_recipe 'kubernetes-cluster::proxy' do 4 | context 'with default node attributes' do 5 | let(:chef_run) { ChefSpec::SoloRunner.new.converge(described_recipe) } 6 | 7 | it { expect(chef_run.node.tags).to eq(['kubernetes.proxy']) } 8 | it { expect(chef_run).to install_yum_package('haproxy >= 1.5.4') } 9 | it { expect(chef_run).to enable_service('haproxy') } 10 | 11 | it 'should create template "/etc/haproxy/haproxy.cfg"' do 12 | expect(chef_run).to create_template('/etc/haproxy/haproxy.cfg').with( 13 | mode: '0644', 14 | source: 'proxy.erb', 15 | variables: { 16 | api_servers: nil, 17 | etcd_client_port: '2379', 18 | kubernetes_api_port: '8080', 19 | kubernetes_secure_api_port: '8443' 20 | } 21 | ) 22 | resource = chef_run.template('/etc/haproxy/haproxy.cfg') 23 | expect(resource).to notify('service[haproxy]').to(:restart).immediately 24 | end 25 | end 26 | end 27 | -------------------------------------------------------------------------------- /test/spec/recipes/registry_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe_recipe 'kubernetes-cluster::registry' do 4 | context 'with default node attributes' do 5 | let(:chef_run) { ChefSpec::SoloRunner.new.converge(described_recipe) } 6 | 7 | it { expect(chef_run.node.tags).to eq(['docker.registry']) } 8 | it { expect(chef_run).to install_yum_package('docker-registry >= 0.9.1') } 9 | it { expect(chef_run).to install_yum_package('cockpit >= 0.71') } 10 | it { expect(chef_run).to enable_service('docker-registry') } 11 | 12 | it 'should create template "/etc/sysconfig/docker-registry"' do 13 | expect(chef_run).to create_template('/etc/sysconfig/docker-registry').with( 14 | mode: '0640', 15 | source: 'docker-registry.erb', 16 | variables: { 17 | registry_port: '5000', 18 | registry_storage: '/var/docker-registry/', 19 | registry_workers: '8' 20 | } 21 | ) 22 | resource = chef_run.template('/etc/sysconfig/docker-registry') 23 | expect(resource).to notify('service[docker-registry]').to(:restart).immediately 24 | end 25 | end 26 | end 27 | -------------------------------------------------------------------------------- /test/spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | require 'chefspec' 2 | require 'chefspec/berkshelf' 3 | require 'chefspec/cacher' 4 | 5 | at_exit { ChefSpec::Coverage.report! } 6 | 7 | module KubernetesCookbook 8 | module SpecHelper 9 | # Allows testing recipes in isolation 10 | def global_stubs_include_recipe 11 | # Don't worry about external cookbook dependencies 12 | allow_any_instance_of(Chef::Cookbook::Metadata).to receive(:depends) 13 | 14 | # Test each recipe in isolation, regardless of includes 15 | @included_recipes = [] 16 | 17 | allow_any_instance_of(Chef::RunContext).to receive(:loaded_recipe?).and_return(false) 18 | allow_any_instance_of(Chef::Recipe).to receive(:include_recipe) do |i| 19 | allow_any_instance_of(Chef::RunContext).to receive(:loaded_recipe?).and_return(true) 20 | @included_recipes << i 21 | end 22 | allow_any_instance_of(Chef::RunContext).to receive(:loaded_recipe).and_return(@included_recipes) 23 | end 24 | end 25 | end 26 | 27 | RSpec.configure do |config| 28 | config.platform = 'redhat' 29 | config.version = '7.1' 30 | 31 | config.color = true 32 | config.alias_example_group_to :describe_recipe, type: :recipe 33 | 34 | Kernel.srand config.seed 35 | config.order = :random 36 | 37 | config.default_formatter = 'doc' if config.files_to_run.one? 38 | 39 | config.expect_with :rspec do |expectations| 40 | expectations.syntax = :expect 41 | end 42 | 43 | config.mock_with :rspec do |mocks| 44 | mocks.syntax = :expect 45 | mocks.verify_partial_doubles = true 46 | end 47 | 48 | config.include KubernetesCookbook::SpecHelper 49 | end 50 | --------------------------------------------------------------------------------