├── .gitignore ├── ISSUE_TEMPLATE.md ├── LICENSE ├── README-cn.md ├── README.md ├── Vagrantfile ├── addon ├── dashboard │ └── kubernetes-dashboard.yaml ├── dns │ ├── coredns.yaml.sed │ └── dns-deploy.sh ├── efk │ ├── es-service.yaml │ ├── es-statefulset.yaml │ ├── fluentd-es-configmap.yaml │ ├── fluentd-es-ds.yaml │ ├── kibana-deployment.yaml │ └── kibana-service.yaml ├── heapster │ ├── grafana-service.yaml │ ├── heapster-controller.yaml │ ├── heapster-service.yaml │ ├── influxdb-grafana-controller.yaml │ └── influxdb-service.yaml ├── istio │ ├── istio-demo.yaml │ └── istio-ingress.yaml ├── jenkins │ └── Dockerfile ├── kiali │ ├── kiali-configmap.yaml │ ├── kiali-secret.yaml │ └── kiali.yaml ├── rook │ ├── mysql.yaml │ ├── rook-agent-clusterrolebinding.yaml │ ├── rook-cluster.yaml │ ├── rook-operator.yaml │ ├── rook-storageclass.yaml │ ├── rook-tools.yaml │ └── wordpress.yaml ├── traefik-ingress │ ├── ingress.yaml │ ├── traefik-rbac.yaml │ └── traefik.yaml ├── vistio │ └── vistio-mesh-only.yaml └── weave-scope │ └── scope.yaml ├── conf ├── admin.kubeconfig ├── apiserver ├── bootstrap.kubeconfig ├── config ├── controller-manager ├── kube-proxy.kubeconfig ├── kubelet.kubeconfig ├── scheduler ├── scheduler.conf └── token.csv ├── hack ├── deploy-base-services.sh ├── deploy-helm.sh ├── get-dashboard-token.sh └── k8s-init.sh ├── images ├── bookinfo-demo.gif ├── dashboard-animation.gif ├── grafana-animation.gif ├── kiali.gif ├── traefik-ingress.gif ├── vistio-animation.gif └── weave-scope-animation.gif ├── install.sh ├── node1 ├── kubelet └── proxy ├── node2 ├── kubelet └── proxy ├── node3 ├── kubelet └── proxy ├── pki ├── admin-csr.json ├── admin-key.pem ├── admin.csr ├── admin.p12 ├── admin.pem ├── ca-config.json ├── ca-csr.json ├── ca-key.pem ├── ca.csr ├── ca.pem ├── kube-proxy-csr.json ├── kube-proxy-key.pem ├── kube-proxy.csr ├── kube-proxy.pem ├── kubernetes-csr.json ├── kubernetes-key.pem ├── kubernetes.csr ├── kubernetes.pem ├── scheduler-csr.json ├── scheduler-key.pem ├── scheduler.csr └── scheduler.pem ├── systemd ├── etcd.service ├── kube-apiserver.service ├── kube-controller-manager.service ├── kube-proxy.service ├── kube-scheduler.service └── kubelet.service ├── yaml ├── admin-role.yaml ├── istio-bookinfo │ ├── bookinfo-gateway.yaml │ ├── bookinfo.yaml │ └── destination-rule-all.yaml └── nginx-deployment.yaml └── yum └── CentOS7-Base-163.repo /.gitignore: -------------------------------------------------------------------------------- 1 | .vagrant/* 2 | .DS_Store 3 | *.tar.gz 4 | kubernetes 5 | *.box 6 | .idea/* 7 | -------------------------------------------------------------------------------- /ISSUE_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | # Environment 2 | 3 | - OS: 4 | - Kubernetes version: 5 | - VirtualBox version: 6 | - Vagrant version: 7 | 8 | ## What I did? 9 | 10 | Your operations on this repo. 11 | 12 | ## Messages 13 | 14 | Logs or error messages. 15 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README-cn.md: -------------------------------------------------------------------------------- 1 | # 使用Vagrant和VirtualBox在本地搭建分布式的Kubernetes集群和Istio Service Mesh 2 | 3 | [Setting up a Kubernetes cluster along with Istio service mesh locally with Vagrant and VirtualBox - English](README.md) 4 | 5 | 当我们需要在本地开发时，更希望能够有一个开箱即用又可以方便定制的分布式开发环境，这样才能对Kubernetes本身和应用进行更好的测试。现在我们使用[Vagrant](https://www.vagrantup.com/)和[VirtualBox](https://www.virtualbox.org/wiki/Downloads)来创建一个这样的环境。 6 | 7 | **注意**：因为使用虚拟机创建分布式Kubernetes集群比较耗费资源，所以我又仅使用Docker创建Standalone的Kubernetes的轻量级[Cloud Native Sandbox](https://github.com/rootsongjc/cloud-native-sandbox)。 8 | 9 | ## Demo 10 | 11 | 点击下面的图片观看视频。 12 | 13 | [![观看视频](https://ae01.alicdn.com/kf/U4416321c10a0444a9f12dd7f5bf722c9p.jpg)](https://www.bilibili.com/video/av39514214/) 14 | 15 | ## 准备环境 16 | 17 | 需要准备以下软件和环境： 18 | 19 | - 8G以上内存 20 | - Vagrant 最新版本（推荐2.2.16） 21 | - VirtualBox 5.2（不支持 5.2 以上的版本） 22 | - 提前下载Kubernetes 1.16（本篇基于1.16.14）的release压缩包 23 | - Mac/Linux，**Windows不完全支持，仅在windows10下通过** 24 | 25 | ## 集群 26 | 27 | 我们使用Vagrant和Virtualbox安装包含3个节点的kubernetes集群，其中master节点同时作为node节点。 28 | 29 | | IP | 主机名 | 组件 | 30 | | ------------ | ----- | ---------------------------------------- | 31 | | 172.17.8.101 | node1 | kube-apiserver、kube-controller-manager、kube-scheduler、etcd、kubelet、docker、flannel、dashboard | 32 | | 172.17.8.102 | node2 | kubelet、docker、flannel、traefik | 33 | | 172.17.8.103 | node3 | kubelet、docker、flannel | 34 | 35 | **注意**：以上的IP、主机名和组件都是固定在这些节点的，即使销毁后下次使用vagrant重建依然保持不变。 36 | 37 | 容器IP范围：172.33.0.0/30 38 | 39 | Kubernetes service IP范围：10.254.0.0/16 40 | 41 | ## 安装的组件 42 | 43 | 安装完成后的集群包含以下组件： 44 | 45 | - flannel（`host-gw`模式） 46 | - kubernetes dashboard 47 | - etcd（单节点） 48 | - kubectl 49 | - CoreDNS 50 | - kubernetes（版本根据下载的kubernetes安装包而定，支持Kubernetes1.9+） 51 | 52 | **可选插件** 53 | 54 | - Heapster + InfluxDB + Grafana 55 | - ElasticSearch + Fluentd + Kibana 56 | - Istio service mesh 57 | - Helm 58 | - Vistio 59 | - Kiali 60 | 61 | ## 使用说明 62 | 63 | 将该repo克隆到本地，下载Kubernetes的到项目的根目录。 64 | 65 | ```bash 66 | git clone https://github.com/rootsongjc/kubernetes-vagrant-centos-cluster.git 67 | cd kubernetes-vagrant-centos-cluster 68 | ``` 69 | 70 | **注意**：如果您是第一次运行该部署程序，那么可以直接执行下面的命令，它将自动帮你下载 Kubernetes 安装包，下一次你就不需要自己下载了，另外您也可以在[这里](https://kubernetes.io/docs/setup/release/notes/)找到Kubernetes的发行版下载地址，下载 Kubernetes发行版后重命名为`kubernetes-server-linux-amd64.tar.gz`，并移动到该项目的根目录下。 71 | 72 | 因为该项目是使用 NFS 的方式挂载到虚拟机的 `/vagrant` 目录中的，所以在安装 NFS 的时候需要您输入密码授权。 73 | 74 | 使用vagrant启动集群。 75 | 76 | ```bash 77 | vagrant up 78 | ``` 79 | 80 | 如果是首次部署，会自动下载`centos/7`的box，这需要花费一些时间，另外每个节点还需要下载安装一系列软件包，整个过程大概需要10几分钟。 81 | 82 | 如果您在运行`vagrant up`的过程中发现无法下载`centos/7`的box，可以手动下载后将其添加到vagrant中。 83 | 84 | **手动添加centos/7 box** 85 | 86 | ````bash 87 | wget -c http://cloud.centos.org/centos/7/vagrant/x86_64/images/CentOS-7-x86_64-Vagrant-1801_02.VirtualBox.box 88 | vagrant box add CentOS-7-x86_64-Vagrant-1804_02.VirtualBox.box --name centos/7 89 | ```` 90 | 91 | 这样下次运行`vagrant up`的时候就会自动读取本地的`centos/7` box而不会再到网上下载。 92 | 93 | #### Mac 安装说明 94 | 95 | 在偏好设置 -> 安全性和隐私 -> 通用，点击被阻止的程序。 96 | 然后在命令行终端中执行 `sudo "/Library/Application Support/VirtualBox/LaunchDaemons/VirtualBoxStartup.sh" restart `， 再执行 `vagrant up` 启动。 97 | 98 | **Windows 安装特别说明** 99 | 100 | 执行`vagrant up`之后会有如下提示： 101 | ``` 102 | G:\code\kubernetes-vagrant-centos-cluster>vagrant up 103 | Bringing machine 'node1' up with 'virtualbox' provider... 104 | Bringing machine 'node2' up with 'virtualbox' provider... 105 | Bringing machine 'node3' up with 'virtualbox' provider... 106 | ==> node1: Importing base box 'centos/7'... 107 | ==> node1: Matching MAC address for NAT networking... 108 | ==> node1: Setting the name of the VM: node1 109 | ==> node1: Clearing any previously set network interfaces... 110 | ==> node1: Specific bridge 'en0: Wi-Fi (AirPort)' not found. You may be asked to specify 111 | ==> node1: which network to bridge to. 112 | ==> node1: Available bridged network interfaces: 113 | 1) Realtek PCIe GBE Family Controller 114 | 2) TAP-Windows Adapter V9 115 | ==> node1: When choosing an interface, it is usually the one that is 116 | ==> node1: being used to connect to the internet. 117 | node1: Which interface should the network bridge to? 118 | node1: Which interface should the network bridge to? 119 | 120 | ``` 121 | 输入`1`之后按回车继续。（根据自己真实网卡选择，node2、node3同样需要） 122 | 123 | 124 | node3快要结束的时候可能会有如下错误： 125 | ``` 126 | node3: Created symlink from /etc/systemd/system/multi-user.target.wants/kubelet.service to /usr/lib/systemd/system/kubelet.service. 127 | node3: Created symlink from /etc/systemd/system/multi-user.target.wants/kube-proxy.service to /usr/lib/systemd/system/kube-proxy.service. 128 | node3: deploy coredns 129 | node3: /tmp/vagrant-shell: ./dns-deploy.sh: /bin/bash^M: bad interpreter: No such file or directory 130 | node3: error: no objects passed to apply 131 | node3: /home/vagrant 132 | ``` 133 | 134 | 解决方法： 135 | 136 | ```bash 137 | vagrant ssh node3 138 | sudo -i 139 | cd /vagrant/addon/dns 140 | yum -y install dos2unix 141 | dos2unix dns-deploy.sh 142 | ./dns-deploy.sh -r 10.254.0.0/16 -i 10.254.0.2 |kubectl apply -f - 143 | ``` 144 | 145 | 146 | ### 访问kubernetes集群 147 | 148 | 访问Kubernetes集群的方式有三种： 149 | 150 | - 本地访问 151 | - 在VM内部访问 152 | - Kubernetes dashboard 153 | 154 | **通过本地访问** 155 | 156 | 可以直接在你自己的本地环境中操作该kubernetes集群，而无需登录到虚拟机中。 157 | 158 | 要想在本地直接操作Kubernetes集群，需要在你的电脑里安装`kubectl`命令行工具，对于Mac用户执行以下步骤： 159 | 160 | ```bash 161 | wget https://storage.googleapis.com/kubernetes-release/release/v1.16.14/kubernetes-client-darwin-amd64.tar.gz 162 | tar xvf kubernetes-client-darwin-amd64.tar.gz && cp kubernetes/client/bin/kubectl /usr/local/bin 163 | ``` 164 | 165 | 将`conf/admin.kubeconfig`文件放到`~/.kube/config`目录下即可在本地使用`kubectl`命令操作集群。 166 | 167 | ```bash 168 | mkdir -p ~/.kube 169 | cp conf/admin.kubeconfig ~/.kube/config 170 | ``` 171 | 172 | 我们推荐您使用这种方式。 173 | 174 | **在虚拟机内部访问** 175 | 176 | 如果有任何问题可以登录到虚拟机内部调试： 177 | 178 | ```bash 179 | vagrant ssh node1 180 | sudo -i 181 | kubectl get nodes 182 | ``` 183 | 184 | **Kubernetes dashboard** 185 | 186 | 还可以直接通过dashboard UI来访问：https://172.17.8.101:8443 187 | 188 | 可以在本地执行以下命令获取token的值（需要提前安装kubectl）： 189 | 190 | ```bash 191 | kubectl -n kube-system describe secret `kubectl -n kube-system get secret|grep admin-token|cut -d " " -f1`|grep "token:"|tr -s " "|cut -d " " -f2 192 | ``` 193 | 194 | **注意**：token的值也可以在`vagrant up`的日志的最后看到。 195 | 196 | ![Kubernetes dashboard animation](images/dashboard-animation.gif) 197 | 198 | 只有当你安装了下面的heapster组件后才能看到上图中的监控metrics。 199 | 200 | 201 | **Windows下Chrome/Firefox访问** 202 | 203 | 如果提示`NET::ERR_CERT_INVALID`，则需要下面的步骤 204 | 205 | 进入本项目目录 206 | 207 | ``` 208 | vagrant ssh node1 209 | sudo -i 210 | cd /vagrant/addon/dashboard/ 211 | mkdir certs 212 | openssl req -nodes -newkey rsa:2048 -keyout certs/dashboard.key -out certs/dashboard.csr -subj "/C=/ST=/L=/O=/OU=/CN=kubernetes-dashboard" 213 | openssl x509 -req -sha256 -days 365 -in certs/dashboard.csr -signkey certs/dashboard.key -out certs/dashboard.crt 214 | kubectl delete secret kubernetes-dashboard-certs -n kube-system 215 | kubectl create secret generic kubernetes-dashboard-certs --from-file=certs -n kube-system 216 | kubectl delete pods $(kubectl get pods -n kube-system|grep kubernetes-dashboard|awk '{print $1}') -n kube-system #重新创建dashboard 217 | ``` 218 | 刷新浏览器之后点击`高级`，选择跳过即可打开页面。 219 | 220 | ### 组件 221 | 222 | **Heapster监控** 223 | 224 | 创建Heapster监控： 225 | 226 | ```bash 227 | kubectl apply -f addon/heapster/ 228 | ``` 229 | 230 | 访问Grafana 231 | 232 | 使用Ingress方式暴露的服务，在本地`/etc/hosts`中增加一条配置： 233 | 234 | ```ini 235 | 172.17.8.102 grafana.jimmysong.io 236 | ``` 237 | 238 | 访问Grafana： 239 | 240 | ![Grafana动画](images/grafana-animation.gif) 241 | 242 | **Traefik** 243 | 244 | 部署Traefik ingress controller和增加ingress配置： 245 | 246 | ```bash 247 | kubectl apply -f addon/traefik-ingress 248 | ``` 249 | 250 | 在本地`/etc/hosts`中增加一条配置： 251 | 252 | ```ini 253 | 172.17.8.102 traefik.jimmysong.io 254 | ``` 255 | 256 | 访问Traefik UI： 257 | 258 | ![Traefik Ingress controller](images/traefik-ingress.gif) 259 | 260 | **EFK** 261 | 262 | 使用EFK做日志收集。 263 | 264 | ```bash 265 | kubectl apply -f addon/efk/ 266 | ``` 267 | 268 | **注意**：运行EFK的每个节点需要消耗很大的CPU和内存，请保证每台虚拟机至少分配了4G内存。 269 | 270 | **Helm** 271 | 272 | 用来部署helm。 273 | 274 | ```bash 275 | hack/deploy-helm.sh 276 | ``` 277 | 278 | ### Service Mesh 279 | 280 | 我们使用 [istio](https://istio.io) 作为 service mesh。 281 | 282 | **安装** 283 | 284 | 到[Istio release](https://github.com/istio/istio/releases) 页面下载istio的安装包，安装istio命令行工具，将`istioctl`命令行工具放到你的`$PATH`目录下，对于Mac用户： 285 | 286 | ```bash 287 | wget https://github.com/istio/istio/releases/download/1.0.0/istio-1.0.0-osx.tar.gz 288 | tar xvf istio-1.0.0-osx.tar.gz 289 | mv bin/istioctl /usr/local/bin/ 290 | ``` 291 | 292 | 在Kubernetes中部署istio： 293 | 294 | ```bash 295 | kubectl apply -f addon/istio/istio-demo.yaml 296 | kubectl apply -f addon/istio/istio-ingress.yaml 297 | ``` 298 | 299 | **运行示例** 300 | 301 | 我们开启了Sidecar自动注入。 302 | 303 | ```bash 304 | kubectl label namespace default istio-injection=enabled 305 | kubectl apply -n default -f yaml/istio-bookinfo/bookinfo.yaml 306 | kubectl apply -n default -f yaml/istio-bookinfo/bookinfo-gateway.yaml 307 | kubectl apply -n default -f yaml/istio-bookinfo/destination-rule-all.yaml 308 | ``` 309 | 310 | 在您自己的本地主机的`/etc/hosts`文件中增加如下配置项。 311 | 312 | ``` 313 | 172.17.8.102 grafana.istio.jimmysong.io 314 | 172.17.8.102 prometheus.istio.jimmysong.io 315 | 172.17.8.102 servicegraph.istio.jimmysong.io 316 | 172.17.8.102 jaeger-query.istio.jimmysong.io 317 | ``` 318 | 319 | 我们可以通过下面的URL地址访问以上的服务。 320 | 321 | | Service | URL | 322 | | ------------ | ------------------------------------------------------------ | 323 | | grafana | http://grafana.istio.jimmysong.io | 324 | | servicegraph | , , | 325 | | tracing | http://jaeger-query.istio.jimmysong.io | 326 | | productpage | http://172.17.8.101:31380/productpage | 327 | 328 | 详细信息请参阅：https://istio.io/zh/docs/examples/bookinfo/ 329 | 330 | ![Bookinfo Demo](images/bookinfo-demo.gif) 331 | 332 | ### Vistio 333 | 334 | [Vizceral](https://github.com/Netflix/vizceral)是Netflix发布的一个开源项目，用于近乎实时地监控应用程序和集群之间的网络流量。Vistio是使用Vizceral对Istio和网格监控的改进。它利用Istio Mixer生成的指标，然后将其输入Prometheus。Vistio查询Prometheus并将数据存储在本地以允许重播流量。 335 | 336 | ```bash 337 | # Deploy vistio via kubectl 338 | kubectl -n default apply -f addon/vistio/ 339 | 340 | # Expose vistio-api 341 | kubectl -n default port-forward $(kubectl -n default get pod -l app=vistio-api -o jsonpath='{.items[0].metadata.name}') 9091:9091 & 342 | 343 | # Expose vistio in another terminal window 344 | kubectl -n default port-forward $(kubectl -n default get pod -l app=vistio-web -o jsonpath='{.items[0].metadata.name}') 8080:8080 & 345 | ``` 346 | 347 | 如果一切都已经启动并准备就绪，您就可以访问Vistio UI，开始探索服务网格网络，访问[http://localhost:8080](http://localhost:8080/) 您将会看到类似下图的输出。 348 | 349 | ![vistio animation](images/vistio-animation.gif) 350 | 351 | 更多详细内容请参考[Vistio—使用Netflix的Vizceral可视化Istio service mesh](http://www.servicemesher.com/blog/vistio-visualize-your-istio-mesh-using-netflixs-vizceral/)。 352 | 353 | ### Kiali 354 | 355 | Kiali是一个用于提供Istio service mesh观察性的项目，更多信息请查看[https://kiali.io](https://kiali.io/)。 356 | 357 | 在本地该项目的根路径下执行下面的命令： 358 | 359 | ```bash 360 | kubectl apply -n istio-system -f addon/kiali 361 | ``` 362 | 363 | Kiali web地址：http://172.17.8.101:32439 364 | 365 | 用户名/密码：admin/admin 366 | 367 | ![kiali](./images/kiali.gif) 368 | 369 | **注意**：Kilia使用Jaeger做追踪，请不用屏蔽kilia页面的弹出窗口。 370 | 371 | ### Weave scope 372 | 373 | [Weave scope](https://github.com/weaveworks/scope)可用于监控、可视化和管理Docker&Kubernetes集群，详情见 374 | 375 | 在本地该项目的根路径下执行下面的命令： 376 | 377 | ```bash 378 | kubectl apply -f addon/weave-scope 379 | ``` 380 | 381 | 在本地的`/etc/hosts`下增加一条记录。 382 | 383 | ``` 384 | 172.17.8.102 scope.weave.jimmysong.io 385 | ``` 386 | 387 | 现在打开浏览器，访问http://scope.weave.jimmysong.io/ 388 | 389 | ![Weave scope动画](images/weave-scope-animation.gif) 390 | 391 | ## 管理 392 | 393 | 除了特别说明，以下命令都在当前的repo目录下操作。 394 | 395 | ### 挂起 396 | 397 | 将当前的虚拟机挂起，以便下次恢复。 398 | 399 | ```bash 400 | vagrant suspend 401 | ``` 402 | 403 | ### 恢复 404 | 405 | 恢复虚拟机的上次状态。 406 | 407 | ```bash 408 | vagrant resume 409 | ``` 410 | 411 | 注意：我们每次挂起虚拟机后再重新启动它们的时候，看到的虚拟机中的时间依然是挂载时候的时间，这样将导致监控查看起来比较麻烦。因此请考虑先停机再重新启动虚拟机。 412 | 413 | ### 重启 414 | 415 | 停机后重启启动。 416 | 417 | ```bash 418 | vagrant halt 419 | vagrant up 420 | # login to node1 421 | vagrant ssh node1 422 | # run the prosivision scripts 423 | /vagrant/hack/k8s-init.sh 424 | exit 425 | # login to node2 426 | vagrant ssh node2 427 | # run the prosivision scripts 428 | /vagrant/hack/k8s-init.sh 429 | exit 430 | # login to node3 431 | vagrant ssh node3 432 | # run the prosivision scripts 433 | /vagrant/hack/k8s-init.sh 434 | sudo -i 435 | cd /vagrant/hack 436 | ./deploy-base-services.sh 437 | exit 438 | ``` 439 | 440 | 现在你已经拥有一个完整的基础的kubernetes运行环境，在该repo的根目录下执行下面的命令可以获取kubernetes dashboard的admin用户的token。 441 | 442 | ```bash 443 | hack/get-dashboard-token.sh 444 | ``` 445 | 446 | 根据提示登录即可。 447 | 448 | ### 清理 449 | 450 | 清理虚拟机。 451 | 452 | ```bash 453 | vagrant destroy 454 | rm -rf .vagrant 455 | ``` 456 | 457 | ### 注意 458 | 459 | 仅做开发测试使用，不要在生产环境使用该项目。 460 | 461 | ## 参考 462 | 463 | - [Kubernetes Handbook——Kubernetes中文指南/云原生应用架构实践手册](https://jimmysong.io/kubernetes-handbook) 464 | - [duffqiu/centos-vagrant](https://github.com/duffqiu/centos-vagrant) 465 | - [coredns/deployment](https://github.com/coredns/deployment) 466 | - [Kubernetes 1.8 kube-proxy 开启 ipvs](https://mritd.me/2017/10/10/kube-proxy-use-ipvs-on-kubernetes-1.8/#%E4%B8%80%E7%8E%AF%E5%A2%83%E5%87%86%E5%A4%87) 467 | - [Vistio—使用Netflix的Vizceral可视化Istio service mesh](http://www.servicemesher.com/blog/vistio-visualize-your-istio-mesh-using-netflixs-vizceral/) 468 | 469 | **更多[Istio](https://istio.io/zh)和Service Mesh的资讯请访问[ServiceMesher社区](http://www.servicemesher.com)和关注社区的微信公众号。** 470 | 471 |

472 | ServiceMesher微信公众号二维码 473 |

474 | 475 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Setting up a distributed Kubernetes cluster along with Istio service mesh locally with Vagrant and VirtualBox 2 | 3 | [使用Vagrant和VirtualBox在本地搭建分布式Kubernetes集群和Istio Service Mesh - 中文](README-cn.md) 4 | 5 | Setting up a Kubernetes cluster and Istio service mesh with `vagrantfile` which consists of 1 master(also as node) and 3 nodes. You don't have to create complicated CA files or configuration. 6 | 7 | **Note**: Because of using virtual machines to setup distributed Kubernetes cluster will bring high load on your computer, so I created the lightweight [Cloud Native Sandbox](https://github.com/rootsongjc/cloud-native-sandbox) using Docker to setup a standalone Kubernetes. 8 | 9 | ## Demo 10 | 11 | Click the following image to watch the video. 12 | 13 | [![Watch the video](https://img.youtube.com/vi/26kbaZxcB4A/maxresdefault.jpg)](https://youtu.be/26kbaZxcB4A) 14 | 15 | ### Why not use kubeadm? 16 | 17 | Because I want to setup the etcd, apiserver, controller and scheduler without docker container. 18 | 19 | ### Architecture 20 | 21 | We will create a Kubernetes 1.15.0 cluster with 3 nodes which contains the components below: 22 | 23 | | IP | Hostname | Componets | 24 | | ------------ | -------- | ---------------------------------------- | 25 | | 172.17.8.101 | node1 | kube-apiserver, kube-controller-manager, kube-scheduler, etcd, kubelet, docker, flannel, dashboard | 26 | | 172.17.8.102 | node2 | kubelet, docker, flannel、traefik | 27 | | 172.17.8.103 | node3 | kubelet, docker, flannel | 28 | 29 | The default setting will create the private network from 172.17.8.101 to 172.17.8.103 for nodes, and it will use the host's DHCP for the public IP. 30 | 31 | The kubernetes service's VIP range is `10.254.0.0/16`. 32 | 33 | The container network range is `170.33.0.0/16` owned by flanneld with `host-gw` backend. 34 | 35 | `kube-proxy` will run as `ipvs` mode. 36 | 37 | ## Usage 38 | 39 | ### Prerequisite 40 | 41 | * Host server with 8G+ mem(More is better), 60G disk, 8 core cpu at lease 42 | * **Vagrant latest（2.2.16 recommended）** 43 | * **VirtualBox 5.2 (5.2+ is not supported)** 44 | * Kubernetes 1.16 (support the latest version 1.16.14) 45 | * Across GFW to download the kubernetes files (For China users only) 46 | * MacOS/Linux (**Windows is not supported completely**) 47 | * NFS Server Package 48 | 49 | ### Support Add-ons 50 | 51 | **Required** 52 | 53 | - CoreDNS 54 | - Dashboard 55 | - Traefik 56 | 57 | **Optional** 58 | 59 | - Heapster + InfluxDB + Grafana 60 | - ElasticSearch + Fluentd + Kibana 61 | - Istio service mesh 62 | - Helm 63 | - Vistio 64 | - Kiali 65 | 66 | #### Setup 67 | 68 | Clone this repo into your local machine and download kubernetes binary release first and move them into the root directory of this repo (GitBash for the Windows must be run as Administrator to install ```vagrant-winnfsd``` plugin). 69 | 70 | ```bash 71 | vagrant plugin install vagrant-winnfsd 72 | git clone https://github.com/rootsongjc/kubernetes-vagrant-centos-cluster.git 73 | cd kubernetes-vagrant-centos-cluster 74 | ``` 75 | 76 | **Note**: If this your first time to setup Kubernetes cluster with vagrant, just skip the above step and run the following command, it will download Kubernetes release automatically for you and no need to download the release next time. You can find the download address the Kubernetes releases [here](https://kubernetes.io/docs/imported/release/notes/). Download the release of version you wanted, move it to the root of this repo, rename it to `kubernetes-server-linux-amd64.tar.gz` then the `install.sh` script will skip the download step. 77 | 78 | As this repo folder is mounted to `/vagrant` with NFS in virtual machines, you may be required to enter a password to for administrator privileges during the installation. 79 | 80 | Set up Kubernetes cluster with vagrant. 81 | 82 | ```bash 83 | vagrant up 84 | ``` 85 | 86 | Wait about 10 minutes the kubernetes cluster will be setup automatically. 87 | 88 | If you have difficult to vagrant up the cluster because of have no way to download the `centos/7` box, you can download the box and add it first. 89 | 90 | **Add centos/7 box manually** 91 | 92 | ```bash 93 | wget -c http://cloud.centos.org/centos/7/vagrant/x86_64/images/CentOS-7-x86_64-Vagrant-1801_02.VirtualBox.box 94 | vagrant box add CentOS-7-x86_64-Vagrant-1804_02.VirtualBox.box --name centos/7 95 | ``` 96 | 97 | The next time you run `vagrant up`, vagrant will import the local box automatically. 98 | 99 | #### Note for Mac 100 | 101 | VirtualBox may be blocked by Mac's security limit. 102 | Go to `System Preferences` - `Security & Privacy` - `Gerneral` click the blocked app and unblock it. 103 | 104 | Run `sudo "/Library/Application Support/VirtualBox/LaunchDaemons/VirtualBoxStartup.sh" restart` in terminal and then `vagrant up`. 105 | 106 | #### Note for Windows 107 | 108 | - The project will run some bash script under the VirtualMachines. These scripts line ending need to be in LF. Git for windows set ```core.autocrlf``` true by default at the installation time. When you clone this project repository, this parameter (set to true) ask git to change all line ending to CRLF. This behavior need to be changed before cloning the repository (or after for each files by hand). We recommend to turn this off by running ```git config --global core.autocrlf false``` and ```git config --global core.eol lf``` before cloning. Then, after cloning, do not forget to turn the behavior back if you want to run other windows projects: ```git config --global core.autocrlf true``` and ```git config --global core.eol crlf```. 109 | 110 | 111 | If you have executed the previous git global configuration then, you will not see these output while node3 is going to be complete: 112 | 113 | ```bash 114 | node3: Created symlink from /etc/systemd/system/multi-user.target.wants/kubelet.service to /usr/lib/systemd/system/kubelet.service. 115 | node3: Created symlink from /etc/systemd/system/multi-user.target.wants/kube-proxy.service to /usr/lib/systemd/system/kube-proxy.service. 116 | node3: deploy coredns 117 | node3: /tmp/vagrant-shell: ./dns-deploy.sh: /bin/bash^M: bad interpreter: No such file or directory 118 | node3: error: no objects passed to apply 119 | node3: /home/vagrant 120 | ``` 121 | 122 | Solution: 123 | 124 | ```bash 125 | vagrant ssh node3 126 | sudo -i 127 | cd /vagrant/addon/dns 128 | yum -y install dos2unix 129 | dos2unix dns-deploy.sh 130 | ./dns-deploy.sh -r 10.254.0.0/16 -i 10.254.0.2 |kubectl apply -f - 131 | ``` 132 | 133 | #### Connect to kubernetes cluster 134 | 135 | There are 3 ways to access the kubernetes cluster. 136 | 137 | - on local 138 | - login to VM 139 | - Kubernetes dashboard 140 | 141 | **local** 142 | 143 | In order to manage the cluster on local you should Install `kubectl` command line tool first(But, you don't need to do it manually because of ```install.sh``` script itself does this). 144 | 145 | Go to [Kubernetes release notes](https://kubernetes.io/docs/setup/release/notes/), download the client binaries, unzip it and then move `kubectl` to your `$PATH` folder, for MacOS: 146 | 147 | ```bash 148 | wget https://storage.googleapis.com/kubernetes-release/release/v1.16.14/kubernetes-client-darwin-amd64.tar.gz 149 | tar xvf kubernetes-client-darwin-amd64.tar.gz && cp kubernetes/client/bin/kubectl /usr/local/bin 150 | ``` 151 | 152 | Copy `conf/admin.kubeconfig` to `~/.kube/config`, using `kubectl` CLI to access the cluster. 153 | 154 | ```bash 155 | mkdir -p ~/.kube 156 | cp conf/admin.kubeconfig ~/.kube/config 157 | ``` 158 | 159 | We recommend you follow this way. 160 | 161 | **VM** 162 | 163 | Login to the virtual machine for debuging. In most situations, you have no need to login the VMs. 164 | 165 | ```bash 166 | vagrant ssh node1 167 | sudo -i 168 | kubectl get nodes 169 | kubectl get pods --namespace=kube-system 170 | ``` 171 | 172 | **Kubernetes dashboard** 173 | 174 | Kubernetes dashboard URL: 175 | 176 | Get the admin token: 177 | 178 | ```bash 179 | kubectl -n kube-system describe secret `kubectl -n kube-system get secret|grep admin-token|cut -d " " -f1`|grep "token:"|tr -s " "|cut -d " " -f2 180 | ``` 181 | 182 | **Note**: You can see the token message on console when `vagrant up` done. 183 | 184 | ![Kubernetes dashboard animation](images/dashboard-animation.gif) 185 | 186 | Only if you install the heapter addon bellow that you can see the metrics. 187 | 188 | **Visit from Chrome/Firefox on Windows** 189 | 190 | If you see the hint `NET::ERR_CERT_INVALID`, follow these steps: 191 | 192 | ```bash 193 | vagrant ssh node1 194 | sudo -i 195 | cd /vagrant/addon/dashboard/ 196 | mkdir certs 197 | openssl req -nodes -newkey rsa:2048 -keyout certs/dashboard.key -out certs/dashboard.csr -subj "/C=/ST=/L=/O=/OU=/CN=kubernetes-dashboard" 198 | openssl x509 -req -sha256 -days 365 -in certs/dashboard.csr -signkey certs/dashboard.key -out certs/dashboard.crt 199 | kubectl delete secret kubernetes-dashboard-certs -n kube-system 200 | kubectl create secret generic kubernetes-dashboard-certs --from-file=certs -n kube-system 201 | kubectl delete pods $(kubectl get pods -n kube-system|grep kubernetes-dashboard|awk '{print $1}') -n kube-system #re-install dashboard 202 | ``` 203 | 204 | Refresh the browser and click `Advance`, skip it. You will see the dashboard page there. 205 | 206 | ## Components 207 | 208 | **Heapster monitoring** 209 | 210 | Run this command on your local machine. 211 | 212 | ```bash 213 | kubectl apply -f /vagrant/addon/heapster/ 214 | ``` 215 | 216 | Append the following item to your local `/etc/hosts` file. 217 | 218 | ```ini 219 | 172.17.8.102 grafana.jimmysong.io 220 | ``` 221 | 222 | Open the URL in browser: 223 | 224 | ![Grafana animation](images/grafana-animation.gif) 225 | 226 | **Traefik** 227 | 228 | Run this command on your local machine. 229 | 230 | ```bash 231 | kubectl apply -f /vagrant/addon/traefik-ingress 232 | ``` 233 | 234 | Append the following item to your local file `/etc/hosts`. 235 | 236 | ```ini 237 | 172.17.8.102 traefik.jimmysong.io 238 | ``` 239 | 240 | Traefik UI URL: 241 | 242 | ![Traefik Ingress controller](images/traefik-ingress.gif) 243 | 244 | **EFK** 245 | 246 | Run this command on your local machine. 247 | 248 | ```bash 249 | kubectl apply -f /vagrant/addon/efk/ 250 | ``` 251 | 252 | **Note**: Powerful CPU and memory allocation required. At least 4G per virtual machine. 253 | 254 | **Helm** 255 | 256 | Run this command on your local machine. 257 | 258 | ```bash 259 | /vagrant/hack/deploy-helm.sh 260 | ``` 261 | 262 | ### Service Mesh 263 | 264 | We use [istio](https://istio.io) as the default service mesh. 265 | 266 | **Installation** 267 | 268 | Go to [Istio release](https://github.com/istio/istio/releases) to download the binary package, install istio command line tool on local and move `istioctl` to your `$PATH` folder, for Mac: 269 | 270 | ```bash 271 | wget https://github.com/istio/istio/releases/download/1.0.0/istio-1.0.0-osx.tar.gz 272 | tar xvf istio-1.0.0-osx.tar.gz 273 | mv istio-1.0.0/bin/istioctl /usr/local/bin/ 274 | ``` 275 | 276 | Deploy istio into Kubernetes: 277 | 278 | ```bash 279 | kubectl apply -f /vagrant/addon/istio/istio-demo.yaml 280 | kubectl apply -f /vagrant/addon/istio/istio-ingress.yaml 281 | ``` 282 | 283 | **Run sample** 284 | 285 | We will let the sidecars be auto injected. 286 | 287 | ```bash 288 | kubectl label namespace default istio-injection=enabled 289 | kubectl apply -n default -f /vagrant/yaml/istio-bookinfo/bookinfo.yaml 290 | kubectl apply -n default -f /vagrant/yaml/istio-bookinfo/bookinfo-gateway.yaml 291 | kubectl apply -n default -f /vagrant/yaml/istio-bookinfo/destination-rule-all.yaml 292 | ``` 293 | 294 | Add the following items into the file `/etc/hosts` of your local machine. 295 | 296 | ``` 297 | 172.17.8.102 grafana.istio.jimmysong.io 298 | 172.17.8.102 prometheus.istio.jimmysong.io 299 | 172.17.8.102 servicegraph.istio.jimmysong.io 300 | 172.17.8.102 jaeger-query.istio.jimmysong.io 301 | ``` 302 | 303 | We can see the services from the following URLs. 304 | 305 | | Service | URL | 306 | | ------------ | ------------------------------------------------------------ | 307 | | grafana | http://grafana.istio.jimmysong.io | 308 | | servicegraph | , , | 309 | | tracing | http://jaeger-query.istio.jimmysong.io | 310 | | productpage | http://172.17.8.101:31380/productpage | 311 | 312 | More detail see https://istio.io/docs/examples/bookinfo/ 313 | 314 | ![Bookinfo Demo](images/bookinfo-demo.gif) 315 | 316 | ### Vistio 317 | 318 | [Vizceral](https://github.com/Netflix/vizceral) is an open source project released by Netflix to monitor network traffic between applications and clusters in near real time. Vistio is an adaptation of Vizceral for Istio and mesh monitoring. It utilizes metrics generated by Istio Mixer which are then fed into Prometheus. Vistio queries Prometheus and stores that data locally to allow for the replaying of traffic. 319 | 320 | Run the following commands in your local machine. 321 | 322 | ```bash 323 | # Deploy vistio via kubectl 324 | kubectl -n default apply -f /vagrant/addon/vistio/ 325 | 326 | # Expose vistio-api 327 | kubectl -n default port-forward $(kubectl -n default get pod -l app=vistio-api -o jsonpath='{.items[0].metadata.name}') 9091:9091 & 328 | 329 | # Expose vistio in another terminal window 330 | kubectl -n default port-forward $(kubectl -n default get pod -l app=vistio-web -o jsonpath='{.items[0].metadata.name}') 8080:8080 & 331 | ``` 332 | 333 | If everything up until now is working you should be able to load the Vistio UI in your browser http://localhost:8080 334 | 335 | ![vistio animation](images/vistio-animation.gif) 336 | 337 | More details see [Vistio — Visualize your Istio Mesh Using Netflix’s Vizceral](https://itnext.io/vistio-visualize-your-istio-mesh-using-netflixs-vizceral-b075c402e18e). 338 | 339 | ### Kiali 340 | 341 | Kiali is a project to help observability for the Istio service mesh, see [https://kiali.io](https://kiali.io/). 342 | 343 | Run the following commands in your local machine. 344 | 345 | ```bash 346 | kubectl apply -n istio-system -f /vagrant/addon/kiali 347 | ``` 348 | 349 | Kiali web: http://172.17.8.101:32439 350 | 351 | User/password: admin/admin 352 | 353 | ![kiali](images/kiali.gif) 354 | 355 | **Note**: Kiali use jaeger for tracing. Do not block the pop-up windows for kiali. 356 | 357 | ### Weave scope 358 | 359 | [Weave scope](https://github.com/weaveworks/scope) is a project for monitoring, visualisation & management for Docker & Kubernetes, see 360 | 361 | Run the following commands in your local machine. 362 | 363 | ```bash 364 | kubectl apply -f /vagrant/addon/weave-scope 365 | ``` 366 | 367 | Add a record on your local `/etc/hosts`. 368 | 369 | ``` 370 | 172.17.8.102 scope.weave.jimmysong.io 371 | ``` 372 | 373 | Now open your browser on http://scope.weave.jimmysong.io/ 374 | 375 | ![Weave scope animation](images/weave-scope-animation.gif) 376 | 377 | ## Operation 378 | 379 | Except for special claim, execute the following commands under the current git repo's root directory. 380 | 381 | ### Suspend 382 | 383 | Suspend the current state of VMs. 384 | 385 | ```bash 386 | vagrant suspend 387 | ``` 388 | 389 | ### Resume 390 | 391 | Resume the last state of VMs. 392 | 393 | ```bash 394 | vagrant resume 395 | ``` 396 | 397 | Note: every time you resume the VMs you will find that the machine time is still at you last time you suspended it. So consider to halt the VMs and restart them. 398 | 399 | ### Restart 400 | 401 | Halt the VMs and up them again. 402 | 403 | ```bash 404 | vagrant halt 405 | vagrant up 406 | # login to node1 407 | vagrant ssh node1 408 | # run the prosivision scripts 409 | /vagrant/hack/k8s-init.sh 410 | exit 411 | # login to node2 412 | vagrant ssh node2 413 | # run the prosivision scripts 414 | /vagrant/hack/k8s-init.sh 415 | exit 416 | # login to node3 417 | vagrant ssh node3 418 | # run the prosivision scripts 419 | /vagrant/hack/k8s-init.sh 420 | sudo -i 421 | cd /vagrant/hack 422 | ./deploy-base-services.sh 423 | exit 424 | ``` 425 | 426 | Now you have provisioned the base kubernetes environments and you can login to kubernetes dashboard, run the following command at the root of this repo to get the admin token. 427 | 428 | ```bash 429 | hack/get-dashboard-token.sh 430 | ``` 431 | 432 | Following the hint to login. 433 | 434 | ### Clean 435 | 436 | Clean up the VMs. 437 | 438 | ```bash 439 | vagrant destroy 440 | rm -rf .vagrant 441 | ``` 442 | 443 | ### Note 444 | 445 | Only use for development and test, don't use it in production environment. 446 | 447 | ## Reference 448 | 449 | * [Kubernetes Handbook - jimmysong.io](https://jimmysong.io/kubernetes-handbook/) 450 | * [duffqiu/centos-vagrant](https://github.com/duffqiu/centos-vagrant) 451 | * [coredns/deployment](https://github.com/coredns/deployment) 452 | * [kubernetes ipvs](https://github.com/kubernetes/kubernetes/tree/master/pkg/proxy/ipvs) 453 | * [Vistio — Visualize your Istio Mesh Using Netflix’s Vizceral](https://itnext.io/vistio-visualize-your-istio-mesh-using-netflixs-vizceral-b075c402e18e) 454 | 455 | **Follow the [ServiceMesher community](http://www.servicemesher.com) on [twitter](https://twitter.com/servicemesher) to get more information about [Istio](https://istio.io) and service mesh.** 456 | -------------------------------------------------------------------------------- /Vagrantfile: -------------------------------------------------------------------------------- 1 | # -*- mode: ruby -*- 2 | # vi: set ft=ruby : 3 | # on win10, you need `vagrant plugin install vagrant-vbguest --plugin-version 0.21` and change synced_folder.type="virtualbox" 4 | # reference `https://www.dissmeyer.com/2020/02/11/issue-with-centos-7-vagrant-boxes-on-windows-10/` 5 | 6 | 7 | Vagrant.configure("2") do |config| 8 | config.vm.box_check_update = false 9 | config.vm.provider 'virtualbox' do |vb| 10 | vb.customize [ "guestproperty", "set", :id, "/VirtualBox/GuestAdd/VBoxService/--timesync-set-threshold", 1000 ] 11 | end 12 | config.vm.synced_folder ".", "/vagrant", type: "virtualbox" 13 | $num_instances = 3 14 | # curl https://discovery.etcd.io/new?size=3 15 | $etcd_cluster = "node1=http://172.17.8.101:2380" 16 | (1..$num_instances).each do |i| 17 | config.vm.define "node#{i}" do |node| 18 | node.vm.box = "centos/7" 19 | node.vbguest.installer_options = { allow_kernel_upgrade: true } 20 | node.vm.box_version = "1804.02" 21 | node.vm.hostname = "node#{i}" 22 | ip = "172.17.8.#{i+100}" 23 | node.vm.network "private_network", ip: ip 24 | node.vm.provider "virtualbox" do |vb| 25 | vb.memory = "3072" 26 | vb.cpus = 1 27 | vb.name = "node#{i}" 28 | end 29 | node.vm.provision "shell", path: "install.sh", args: [i, ip, $etcd_cluster] 30 | end 31 | end 32 | end 33 | -------------------------------------------------------------------------------- /addon/dashboard/kubernetes-dashboard.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2017 The Kubernetes Authors. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # Configuration to deploy release version of the Dashboard UI compatible with 16 | # Kubernetes 1.8. 17 | # 18 | # Example usage: kubectl create -f 19 | 20 | # ------------------- Dashboard Secret ------------------- # 21 | 22 | apiVersion: v1 23 | kind: Secret 24 | metadata: 25 | labels: 26 | k8s-app: kubernetes-dashboard 27 | name: kubernetes-dashboard-certs 28 | namespace: kube-system 29 | type: Opaque 30 | 31 | --- 32 | # ------------------- Dashboard Service Account ------------------- # 33 | 34 | apiVersion: v1 35 | kind: ServiceAccount 36 | metadata: 37 | labels: 38 | k8s-app: kubernetes-dashboard 39 | name: kubernetes-dashboard 40 | namespace: kube-system 41 | 42 | --- 43 | # ------------------- Dashboard Role & Role Binding ------------------- # 44 | 45 | kind: Role 46 | apiVersion: rbac.authorization.k8s.io/v1 47 | metadata: 48 | name: kubernetes-dashboard-minimal 49 | namespace: kube-system 50 | rules: 51 | # Allow Dashboard to create 'kubernetes-dashboard-key-holder' secret. 52 | - apiGroups: [""] 53 | resources: ["secrets"] 54 | verbs: ["create"] 55 | # Allow Dashboard to create 'kubernetes-dashboard-settings' config map. 56 | - apiGroups: [""] 57 | resources: ["configmaps"] 58 | verbs: ["create"] 59 | # Allow Dashboard to get, update and delete Dashboard exclusive secrets. 60 | - apiGroups: [""] 61 | resources: ["secrets"] 62 | resourceNames: ["kubernetes-dashboard-key-holder", "kubernetes-dashboard-certs"] 63 | verbs: ["get", "update", "delete"] 64 | # Allow Dashboard to get and update 'kubernetes-dashboard-settings' config map. 65 | - apiGroups: [""] 66 | resources: ["configmaps"] 67 | resourceNames: ["kubernetes-dashboard-settings"] 68 | verbs: ["get", "update"] 69 | # Allow Dashboard to get metrics from heapster. 70 | - apiGroups: [""] 71 | resources: ["services"] 72 | resourceNames: ["heapster"] 73 | verbs: ["proxy"] 74 | - apiGroups: [""] 75 | resources: ["services/proxy"] 76 | resourceNames: ["heapster", "http:heapster:", "https:heapster:"] 77 | verbs: ["get"] 78 | 79 | --- 80 | apiVersion: rbac.authorization.k8s.io/v1 81 | kind: RoleBinding 82 | metadata: 83 | name: kubernetes-dashboard-minimal 84 | namespace: kube-system 85 | roleRef: 86 | apiGroup: rbac.authorization.k8s.io 87 | kind: Role 88 | name: kubernetes-dashboard-minimal 89 | subjects: 90 | - kind: ServiceAccount 91 | name: kubernetes-dashboard 92 | namespace: kube-system 93 | 94 | --- 95 | # ------------------- Dashboard Deployment ------------------- # 96 | 97 | kind: Deployment 98 | apiVersion: apps/v1 99 | metadata: 100 | labels: 101 | k8s-app: kubernetes-dashboard 102 | name: kubernetes-dashboard 103 | namespace: kube-system 104 | spec: 105 | replicas: 1 106 | revisionHistoryLimit: 10 107 | selector: 108 | matchLabels: 109 | k8s-app: kubernetes-dashboard 110 | template: 111 | metadata: 112 | labels: 113 | k8s-app: kubernetes-dashboard 114 | spec: 115 | hostNetwork: true 116 | nodeSelector: 117 | kubernetes.io/hostname: "node1" 118 | containers: 119 | - name: kubernetes-dashboard 120 | image: registry.aliyuncs.com/google_containers/kubernetes-dashboard-amd64:v1.8.3 121 | ports: 122 | - containerPort: 8443 123 | protocol: TCP 124 | hostPort: 8443 125 | args: 126 | - --auto-generate-certificates 127 | # Uncomment the following line to manually specify Kubernetes API server Host 128 | # If not specified, Dashboard will attempt to auto discover the API server and connect 129 | # to it. Uncomment only if the default does not work. 130 | # - --apiserver-host=http://my-address:port 131 | volumeMounts: 132 | - name: kubernetes-dashboard-certs 133 | mountPath: /certs 134 | # Create on-disk volume to store exec logs 135 | - mountPath: /tmp 136 | name: tmp-volume 137 | livenessProbe: 138 | httpGet: 139 | scheme: HTTPS 140 | path: / 141 | port: 8443 142 | initialDelaySeconds: 30 143 | timeoutSeconds: 30 144 | volumes: 145 | - name: kubernetes-dashboard-certs 146 | secret: 147 | secretName: kubernetes-dashboard-certs 148 | - name: tmp-volume 149 | emptyDir: {} 150 | serviceAccountName: kubernetes-dashboard 151 | # Comment the following tolerations if Dashboard must not be deployed on master 152 | tolerations: 153 | - key: node-role.kubernetes.io/master 154 | effect: NoSchedule 155 | 156 | --- 157 | # ------------------- Dashboard Service ------------------- # 158 | 159 | kind: Service 160 | apiVersion: v1 161 | metadata: 162 | labels: 163 | k8s-app: kubernetes-dashboard 164 | name: kubernetes-dashboard 165 | namespace: kube-system 166 | spec: 167 | type: NodePort 168 | ports: 169 | - port: 8443 170 | targetPort: 8443 171 | nodePort: 30000 172 | selector: 173 | k8s-app: kubernetes-dashboard 174 | -------------------------------------------------------------------------------- /addon/dns/coredns.yaml.sed: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | name: coredns 5 | namespace: kube-system 6 | --- 7 | apiVersion: rbac.authorization.k8s.io/v1 8 | kind: ClusterRole 9 | metadata: 10 | labels: 11 | kubernetes.io/bootstrapping: rbac-defaults 12 | name: system:coredns 13 | rules: 14 | - apiGroups: 15 | - "" 16 | resources: 17 | - endpoints 18 | - services 19 | - pods 20 | - namespaces 21 | verbs: 22 | - list 23 | - watch 24 | --- 25 | apiVersion: rbac.authorization.k8s.io/v1 26 | kind: ClusterRoleBinding 27 | metadata: 28 | annotations: 29 | rbac.authorization.kubernetes.io/autoupdate: "true" 30 | labels: 31 | kubernetes.io/bootstrapping: rbac-defaults 32 | name: system:coredns 33 | roleRef: 34 | apiGroup: rbac.authorization.k8s.io 35 | kind: ClusterRole 36 | name: system:coredns 37 | subjects: 38 | - kind: ServiceAccount 39 | name: coredns 40 | namespace: kube-system 41 | --- 42 | apiVersion: v1 43 | kind: ConfigMap 44 | metadata: 45 | name: coredns 46 | namespace: kube-system 47 | data: 48 | Corefile: | 49 | .:53 { 50 | errors 51 | health 52 | kubernetes CLUSTER_DOMAIN REVERSE_CIDRS { 53 | pods insecure 54 | upstream 55 | fallthrough in-addr.arpa ip6.arpa 56 | } 57 | prometheus :9153 58 | proxy . /etc/resolv.conf 59 | cache 30 60 | reload 61 | loadbalance 62 | } 63 | --- 64 | apiVersion: apps/v1 65 | kind: Deployment 66 | metadata: 67 | name: coredns 68 | namespace: kube-system 69 | labels: 70 | k8s-app: kube-dns 71 | kubernetes.io/name: "CoreDNS" 72 | spec: 73 | replicas: 2 74 | strategy: 75 | type: RollingUpdate 76 | rollingUpdate: 77 | maxUnavailable: 1 78 | selector: 79 | matchLabels: 80 | k8s-app: kube-dns 81 | template: 82 | metadata: 83 | labels: 84 | k8s-app: kube-dns 85 | spec: 86 | serviceAccountName: coredns 87 | tolerations: 88 | - key: "CriticalAddonsOnly" 89 | operator: "Exists" 90 | containers: 91 | - name: coredns 92 | image: coredns/coredns:1.2.0 93 | imagePullPolicy: IfNotPresent 94 | args: [ "-conf", "/etc/coredns/Corefile" ] 95 | volumeMounts: 96 | - name: config-volume 97 | mountPath: /etc/coredns 98 | readOnly: true 99 | ports: 100 | - containerPort: 53 101 | name: dns 102 | protocol: UDP 103 | - containerPort: 53 104 | name: dns-tcp 105 | protocol: TCP 106 | - containerPort: 9153 107 | name: metrics 108 | protocol: TCP 109 | livenessProbe: 110 | httpGet: 111 | path: /health 112 | port: 8080 113 | scheme: HTTP 114 | initialDelaySeconds: 60 115 | timeoutSeconds: 5 116 | successThreshold: 1 117 | failureThreshold: 5 118 | dnsPolicy: Default 119 | volumes: 120 | - name: config-volume 121 | configMap: 122 | name: coredns 123 | items: 124 | - key: Corefile 125 | path: Corefile 126 | --- 127 | apiVersion: v1 128 | kind: Service 129 | metadata: 130 | name: kube-dns 131 | namespace: kube-system 132 | annotations: 133 | prometheus.io/port: "9153" 134 | prometheus.io/scrape: "true" 135 | labels: 136 | k8s-app: kube-dns 137 | kubernetes.io/cluster-service: "true" 138 | kubernetes.io/name: "CoreDNS" 139 | spec: 140 | selector: 141 | k8s-app: kube-dns 142 | clusterIP: CLUSTER_DNS_IP 143 | ports: 144 | - name: dns 145 | port: 53 146 | protocol: UDP 147 | - name: dns-tcp 148 | port: 53 149 | protocol: TCP 150 | -------------------------------------------------------------------------------- /addon/dns/dns-deploy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Deploys CoreDNS to a cluster currently running Kube-DNS. 4 | 5 | show_help () { 6 | cat << USAGE 7 | usage: $0 [ -r REVERSE-CIDR ] [ -i DNS-IP ] [ -d CLUSTER-DOMAIN ] [ -t YAML-TEMPLATE ] 8 | -r : Define a reverse zone for the given CIDR. You may specifcy this option more 9 | than once to add multiple reverse zones. If no reverse CIDRs are defined, 10 | then the default is to handle all reverse zones (i.e. in-addr.arpa and ip6.arpa) 11 | -i : Specify the cluster DNS IP address. If not specificed, the IP address of 12 | the existing "kube-dns" service is used, if present. 13 | USAGE 14 | exit 0 15 | } 16 | 17 | # Simple Defaults 18 | CLUSTER_DOMAIN=cluster.local 19 | YAML_TEMPLATE=`pwd`/coredns.yaml.sed 20 | 21 | 22 | # Get Opts 23 | while getopts "hr:i:d:t:" opt; do 24 | case "$opt" in 25 | h) show_help 26 | ;; 27 | r) REVERSE_CIDRS="$REVERSE_CIDRS $OPTARG" 28 | ;; 29 | i) CLUSTER_DNS_IP=$OPTARG 30 | ;; 31 | d) CLUSTER_DOMAIN=$OPTARG 32 | ;; 33 | t) YAML_TEMPLATE=$OPTARG 34 | ;; 35 | esac 36 | done 37 | 38 | # Conditional Defaults 39 | if [[ -z $REVERSE_CIDRS ]]; then 40 | REVERSE_CIDRS="in-addr.arpa ip6.arpa" 41 | fi 42 | if [[ -z $CLUSTER_DNS_IP ]]; then 43 | # Default IP to kube-dns IP 44 | CLUSTER_DNS_IP=$(kubectl get service --namespace kube-system kube-dns -o jsonpath="{.spec.clusterIP}") 45 | if [ $? -ne 0 ]; then 46 | >&2 echo "Error! The IP address for DNS service couldn't be determined automatically. Please specify the DNS-IP with the '-i' option." 47 | exit 2 48 | fi 49 | fi 50 | 51 | sed -e s/CLUSTER_DNS_IP/$CLUSTER_DNS_IP/g -e s/CLUSTER_DOMAIN/$CLUSTER_DOMAIN/g -e "s?REVERSE_CIDRS?$REVERSE_CIDRS?g" $YAML_TEMPLATE 52 | -------------------------------------------------------------------------------- /addon/efk/es-service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: elasticsearch-logging 5 | namespace: kube-system 6 | labels: 7 | k8s-app: elasticsearch-logging 8 | kubernetes.io/cluster-service: "true" 9 | addonmanager.kubernetes.io/mode: Reconcile 10 | kubernetes.io/name: "Elasticsearch" 11 | spec: 12 | ports: 13 | - port: 9200 14 | protocol: TCP 15 | targetPort: db 16 | selector: 17 | k8s-app: elasticsearch-logging 18 | -------------------------------------------------------------------------------- /addon/efk/es-statefulset.yaml: -------------------------------------------------------------------------------- 1 | # RBAC authn and authz 2 | apiVersion: v1 3 | kind: ServiceAccount 4 | metadata: 5 | name: elasticsearch-logging 6 | namespace: kube-system 7 | labels: 8 | k8s-app: elasticsearch-logging 9 | kubernetes.io/cluster-service: "true" 10 | addonmanager.kubernetes.io/mode: Reconcile 11 | --- 12 | kind: ClusterRole 13 | apiVersion: rbac.authorization.k8s.io/v1 14 | metadata: 15 | name: elasticsearch-logging 16 | labels: 17 | k8s-app: elasticsearch-logging 18 | kubernetes.io/cluster-service: "true" 19 | addonmanager.kubernetes.io/mode: Reconcile 20 | rules: 21 | - apiGroups: 22 | - "" 23 | resources: 24 | - "services" 25 | - "namespaces" 26 | - "endpoints" 27 | verbs: 28 | - "get" 29 | --- 30 | kind: ClusterRoleBinding 31 | apiVersion: rbac.authorization.k8s.io/v1 32 | metadata: 33 | namespace: kube-system 34 | name: elasticsearch-logging 35 | labels: 36 | k8s-app: elasticsearch-logging 37 | kubernetes.io/cluster-service: "true" 38 | addonmanager.kubernetes.io/mode: Reconcile 39 | subjects: 40 | - kind: ServiceAccount 41 | name: elasticsearch-logging 42 | namespace: kube-system 43 | apiGroup: "" 44 | roleRef: 45 | kind: ClusterRole 46 | name: elasticsearch-logging 47 | apiGroup: "" 48 | --- 49 | # Elasticsearch deployment itself 50 | apiVersion: apps/v1 51 | kind: StatefulSet 52 | metadata: 53 | name: elasticsearch-logging 54 | namespace: kube-system 55 | labels: 56 | k8s-app: elasticsearch-logging 57 | version: v5.6.4 58 | kubernetes.io/cluster-service: "true" 59 | addonmanager.kubernetes.io/mode: Reconcile 60 | spec: 61 | serviceName: elasticsearch-logging 62 | replicas: 2 63 | selector: 64 | matchLabels: 65 | k8s-app: elasticsearch-logging 66 | version: v5.6.4 67 | template: 68 | metadata: 69 | labels: 70 | k8s-app: elasticsearch-logging 71 | version: v5.6.4 72 | kubernetes.io/cluster-service: "true" 73 | spec: 74 | serviceAccountName: elasticsearch-logging 75 | containers: 76 | - image: jimmysong/elasticsearch:v5.6.4 77 | name: elasticsearch-logging 78 | resources: 79 | # need more cpu upon initialization, therefore burstable class 80 | limits: 81 | cpu: 1000m 82 | requests: 83 | cpu: 100m 84 | ports: 85 | - containerPort: 9200 86 | name: db 87 | protocol: TCP 88 | - containerPort: 9300 89 | name: transport 90 | protocol: TCP 91 | volumeMounts: 92 | - name: elasticsearch-logging 93 | mountPath: /data 94 | env: 95 | - name: "NAMESPACE" 96 | valueFrom: 97 | fieldRef: 98 | fieldPath: metadata.namespace 99 | volumes: 100 | - name: elasticsearch-logging 101 | emptyDir: {} 102 | # Elasticsearch requires vm.max_map_count to be at least 262144. 103 | # If your OS already sets up this number to a higher value, feel free 104 | # to remove this init container. 105 | initContainers: 106 | - image: alpine:3.6 107 | command: ["/sbin/sysctl", "-w", "vm.max_map_count=262144"] 108 | name: elasticsearch-logging-init 109 | securityContext: 110 | privileged: true 111 | -------------------------------------------------------------------------------- /addon/efk/fluentd-es-configmap.yaml: -------------------------------------------------------------------------------- 1 | kind: ConfigMap 2 | apiVersion: v1 3 | metadata: 4 | name: fluentd-es-config-v0.1.3 5 | namespace: kube-system 6 | labels: 7 | addonmanager.kubernetes.io/mode: Reconcile 8 | data: 9 | system.conf: |- 10 | 11 | root_dir /tmp/fluentd-buffers/ 12 | 13 | 14 | containers.input.conf: |- 15 | # This configuration file for Fluentd / td-agent is used 16 | # to watch changes to Docker log files. The kubelet creates symlinks that 17 | # capture the pod name, namespace, container name & Docker container ID 18 | # to the docker logs for pods in the /var/log/containers directory on the host. 19 | # If running this fluentd configuration in a Docker container, the /var/log 20 | # directory should be mounted in the container. 21 | # 22 | # These logs are then submitted to Elasticsearch which assumes the 23 | # installation of the fluent-plugin-elasticsearch & the 24 | # fluent-plugin-kubernetes_metadata_filter plugins. 25 | # See https://github.com/uken/fluent-plugin-elasticsearch & 26 | # https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter for 27 | # more information about the plugins. 28 | # 29 | # Example 30 | # ======= 31 | # A line in the Docker log file might look like this JSON: 32 | # 33 | # {"log":"2014/09/25 21:15:03 Got request with path wombat\n", 34 | # "stream":"stderr", 35 | # "time":"2014-09-25T21:15:03.499185026Z"} 36 | # 37 | # The time_format specification below makes sure we properly 38 | # parse the time format produced by Docker. This will be 39 | # submitted to Elasticsearch and should appear like: 40 | # $ curl 'http://elasticsearch-logging:9200/_search?pretty' 41 | # ... 42 | # { 43 | # "_index" : "logstash-2014.09.25", 44 | # "_type" : "fluentd", 45 | # "_id" : "VBrbor2QTuGpsQyTCdfzqA", 46 | # "_score" : 1.0, 47 | # "_source":{"log":"2014/09/25 22:45:50 Got request with path wombat\n", 48 | # "stream":"stderr","tag":"docker.container.all", 49 | # "@timestamp":"2014-09-25T22:45:50+00:00"} 50 | # }, 51 | # ... 52 | # 53 | # The Kubernetes fluentd plugin is used to write the Kubernetes metadata to the log 54 | # record & add labels to the log record if properly configured. This enables users 55 | # to filter & search logs on any metadata. 56 | # For example a Docker container's logs might be in the directory: 57 | # 58 | # /var/lib/docker/containers/997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b 59 | # 60 | # and in the file: 61 | # 62 | # 997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b-json.log 63 | # 64 | # where 997599971ee6... is the Docker ID of the running container. 65 | # The Kubernetes kubelet makes a symbolic link to this file on the host machine 66 | # in the /var/log/containers directory which includes the pod name and the Kubernetes 67 | # container name: 68 | # 69 | # synthetic-logger-0.25lps-pod_default_synth-lgr-997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b.log 70 | # -> 71 | # /var/lib/docker/containers/997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b/997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b-json.log 72 | # 73 | # The /var/log directory on the host is mapped to the /var/log directory in the container 74 | # running this instance of Fluentd and we end up collecting the file: 75 | # 76 | # /var/log/containers/synthetic-logger-0.25lps-pod_default_synth-lgr-997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b.log 77 | # 78 | # This results in the tag: 79 | # 80 | # var.log.containers.synthetic-logger-0.25lps-pod_default_synth-lgr-997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b.log 81 | # 82 | # The Kubernetes fluentd plugin is used to extract the namespace, pod name & container name 83 | # which are added to the log message as a kubernetes field object & the Docker container ID 84 | # is also added under the docker field object. 85 | # The final tag is: 86 | # 87 | # kubernetes.var.log.containers.synthetic-logger-0.25lps-pod_default_synth-lgr-997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b.log 88 | # 89 | # And the final log record look like: 90 | # 91 | # { 92 | # "log":"2014/09/25 21:15:03 Got request with path wombat\n", 93 | # "stream":"stderr", 94 | # "time":"2014-09-25T21:15:03.499185026Z", 95 | # "kubernetes": { 96 | # "namespace": "default", 97 | # "pod_name": "synthetic-logger-0.25lps-pod", 98 | # "container_name": "synth-lgr" 99 | # }, 100 | # "docker": { 101 | # "container_id": "997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b" 102 | # } 103 | # } 104 | # 105 | # This makes it easier for users to search for logs by pod name or by 106 | # the name of the Kubernetes container regardless of how many times the 107 | # Kubernetes pod has been restarted (resulting in a several Docker container IDs). 108 | 109 | # Json Log Example: 110 | # {"log":"[info:2016-02-16T16:04:05.930-08:00] Some log text here\n","stream":"stdout","time":"2016-02-17T00:04:05.931087621Z"} 111 | # CRI Log Example: 112 | # 2016-02-17T00:04:05.931087621Z stdout F [info:2016-02-16T16:04:05.930-08:00] Some log text here 113 | 114 | @id fluentd-containers.log 115 | @type tail 116 | path /var/log/containers/*.log 117 | pos_file /var/log/es-containers.log.pos 118 | time_format %Y-%m-%dT%H:%M:%S.%NZ 119 | tag raw.kubernetes.* 120 | format json 121 | read_from_head true 122 | 123 | 124 | # Detect exceptions in the log output and forward them as one log entry. 125 | 126 | @id raw.kubernetes 127 | @type detect_exceptions 128 | remove_tag_prefix raw 129 | message log 130 | stream stream 131 | multiline_flush_interval 5 132 | max_bytes 500000 133 | max_lines 1000 134 | 135 | 136 | system.input.conf: |- 137 | # Example: 138 | # 2015-12-21 23:17:22,066 [salt.state ][INFO ] Completed state [net.ipv4.ip_forward] at time 23:17:22.066081 139 | 140 | @id minion 141 | @type tail 142 | format /^(?