├── provisioning-playbook ├── main.retry ├── roles │ └── common │ │ ├── meta │ │ └── main.yml │ │ ├── defaults │ │ └── main.yml │ │ ├── files │ │ └── main.yml │ │ ├── handlers │ │ └── main.yml │ │ ├── templates │ │ └── main.yml │ │ ├── tasks │ │ ├── main.yml │ │ └── provisioning.yml │ │ └── vars │ │ └── main.yml ├── main.yml └── hosts ├── deploy-app-v1-playbook ├── main.retry ├── roles │ └── common │ │ ├── defaults │ │ └── main.yml │ │ ├── handlers │ │ └── main.yml │ │ ├── meta │ │ └── main.yml │ │ ├── vars │ │ └── main.yml │ │ ├── templates │ │ └── main.yml │ │ ├── tasks │ │ ├── main.yml │ │ └── deploy-app.yml │ │ └── files │ │ ├── service-app.yml │ │ └── app-v1.yml ├── main.yml └── hosts ├── deploy-app-v2-playbook ├── roles │ └── common │ │ ├── defaults │ │ └── main.yml │ │ ├── handlers │ │ └── main.yml │ │ ├── meta │ │ └── main.yml │ │ ├── vars │ │ └── main.yml │ │ ├── templates │ │ └── main.yml │ │ ├── tasks │ │ ├── main.yml │ │ └── deploy-app.yml │ │ └── files │ │ ├── app-v1.yml │ │ └── app-v2.yml ├── main.yml └── hosts ├── k8s-install-playbook ├── roles │ ├── install-helm │ │ ├── meta │ │ │ └── main.yml │ │ ├── defaults │ │ │ └── main.yml │ │ ├── tasks │ │ │ ├── main.yml │ │ │ ├── install-monit-tools.yml │ │ │ └── install-helm.yml │ │ ├── handlers │ │ │ └── main.yml │ │ └── vars │ │ │ └── main.yml │ ├── install_k8s │ │ ├── meta │ │ │ └── main.yml │ │ ├── defaults │ │ │ └── main.yml │ │ ├── vars │ │ │ └── main.yml │ │ ├── tasks │ │ │ ├── main.yml │ │ │ └── install.yml │ │ ├── handlers │ │ │ └── main.yml │ │ ├── templates │ │ │ └── docker-compose.yml.j2 │ │ └── files │ │ │ └── my.cnf │ ├── join-workers │ │ ├── meta │ │ │ └── main.yml │ │ ├── defaults │ │ │ └── main.yml │ │ ├── vars │ │ │ └── main.yml │ │ ├── tasks │ │ │ ├── main.yml │ │ │ └── join-cluster.yml │ │ ├── handlers │ │ │ └── main.yml │ │ ├── templates │ │ │ └── docker-compose.yml.j2 │ │ └── files │ │ │ └── my.cnf │ └── create-cluster │ │ ├── meta │ │ └── main.yml │ │ ├── defaults │ │ └── main.yml │ │ ├── tasks │ │ ├── main.yml │ │ └── init-cluster.yml │ │ ├── vars │ │ └── main.yml │ │ ├── handlers │ │ └── main.yml │ │ └── files │ │ └── my.cnf ├── main.retry ├── hosts └── main.yml ├── canary-deploy-app-v2-playbook ├── roles │ └── common │ │ ├── meta │ │ └── main.yml │ │ ├── vars │ │ └── main.yml │ │ ├── defaults │ │ └── main.yml │ │ ├── handlers │ │ └── main.yml │ │ ├── templates │ │ └── main.yml │ │ ├── tasks │ │ ├── main.yml │ │ └── deploy-app.yml │ │ └── files │ │ └── app-v2-canary.yml ├── main.yml └── hosts ├── images ├── dash-1.png ├── dash-2.png ├── dash-3.png ├── dash.png └── dash-3.1.png ├── nginx-prometheus-exporter-v1 ├── Dockerfile ├── nginx.conf ├── metrics.vhost └── lib │ └── prometheus.lua ├── nginx-prometheus-exporter-v2 ├── Dockerfile ├── nginx.conf ├── metrics.vhost └── lib │ └── prometheus.lua └── README.md /provisioning-playbook/main.retry: -------------------------------------------------------------------------------- 1 | localhost 2 | -------------------------------------------------------------------------------- /provisioning-playbook/roles/common/meta/main.yml: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /deploy-app-v1-playbook/main.retry: -------------------------------------------------------------------------------- 1 | 3.86.229.25 2 | -------------------------------------------------------------------------------- /deploy-app-v1-playbook/roles/common/defaults/main.yml: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /deploy-app-v1-playbook/roles/common/handlers/main.yml: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /deploy-app-v1-playbook/roles/common/meta/main.yml: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /deploy-app-v1-playbook/roles/common/vars/main.yml: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /deploy-app-v2-playbook/roles/common/defaults/main.yml: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /deploy-app-v2-playbook/roles/common/handlers/main.yml: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /deploy-app-v2-playbook/roles/common/meta/main.yml: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /deploy-app-v2-playbook/roles/common/vars/main.yml: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /k8s-install-playbook/roles/install-helm/meta/main.yml: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /k8s-install-playbook/roles/install_k8s/meta/main.yml: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /k8s-install-playbook/roles/join-workers/meta/main.yml: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /provisioning-playbook/roles/common/defaults/main.yml: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /provisioning-playbook/roles/common/files/main.yml: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /provisioning-playbook/roles/common/handlers/main.yml: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /provisioning-playbook/roles/common/templates/main.yml: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /canary-deploy-app-v2-playbook/roles/common/meta/main.yml: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /canary-deploy-app-v2-playbook/roles/common/vars/main.yml: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /deploy-app-v1-playbook/roles/common/templates/main.yml: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /deploy-app-v2-playbook/roles/common/templates/main.yml: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /k8s-install-playbook/roles/create-cluster/meta/main.yml: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /k8s-install-playbook/roles/install-helm/defaults/main.yml: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /k8s-install-playbook/roles/install_k8s/defaults/main.yml: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /k8s-install-playbook/roles/install_k8s/vars/main.yml: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /k8s-install-playbook/roles/join-workers/defaults/main.yml: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /k8s-install-playbook/roles/join-workers/vars/main.yml: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /canary-deploy-app-v2-playbook/roles/common/defaults/main.yml: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /canary-deploy-app-v2-playbook/roles/common/handlers/main.yml: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /canary-deploy-app-v2-playbook/roles/common/templates/main.yml: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /k8s-install-playbook/roles/create-cluster/defaults/main.yml: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /deploy-app-v1-playbook/roles/common/tasks/main.yml: -------------------------------------------------------------------------------- 1 | - include: deploy-app.yml -------------------------------------------------------------------------------- /deploy-app-v2-playbook/roles/common/tasks/main.yml: -------------------------------------------------------------------------------- 1 | - include: deploy-app.yml -------------------------------------------------------------------------------- /k8s-install-playbook/roles/install_k8s/tasks/main.yml: -------------------------------------------------------------------------------- 1 | - include: install.yml -------------------------------------------------------------------------------- /provisioning-playbook/roles/common/tasks/main.yml: -------------------------------------------------------------------------------- 1 | - include: provisioning.yml -------------------------------------------------------------------------------- /canary-deploy-app-v2-playbook/roles/common/tasks/main.yml: -------------------------------------------------------------------------------- 1 | - include: deploy-app.yml -------------------------------------------------------------------------------- /k8s-install-playbook/main.retry: -------------------------------------------------------------------------------- 1 | 3.84.19.167 2 | 3.88.252.202 3 | 34.238.235.66 4 | -------------------------------------------------------------------------------- /k8s-install-playbook/roles/create-cluster/tasks/main.yml: -------------------------------------------------------------------------------- 1 | - include: init-cluster.yml -------------------------------------------------------------------------------- /k8s-install-playbook/roles/join-workers/tasks/main.yml: -------------------------------------------------------------------------------- 1 | - include: join-cluster.yml -------------------------------------------------------------------------------- /provisioning-playbook/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - hosts: local 3 | roles: 4 | - common 5 | -------------------------------------------------------------------------------- /images/dash-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/badtuxx/k8s-canary-deploy-example/HEAD/images/dash-1.png -------------------------------------------------------------------------------- /images/dash-2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/badtuxx/k8s-canary-deploy-example/HEAD/images/dash-2.png -------------------------------------------------------------------------------- /images/dash-3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/badtuxx/k8s-canary-deploy-example/HEAD/images/dash-3.png -------------------------------------------------------------------------------- /images/dash.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/badtuxx/k8s-canary-deploy-example/HEAD/images/dash.png -------------------------------------------------------------------------------- /images/dash-3.1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/badtuxx/k8s-canary-deploy-example/HEAD/images/dash-3.1.png -------------------------------------------------------------------------------- /deploy-app-v1-playbook/main.yml: -------------------------------------------------------------------------------- 1 | - hosts: k8s-master 2 | become: yes 3 | user: ubuntu 4 | roles: 5 | - common -------------------------------------------------------------------------------- /deploy-app-v2-playbook/main.yml: -------------------------------------------------------------------------------- 1 | - hosts: k8s-master 2 | become: yes 3 | user: ubuntu 4 | roles: 5 | - common -------------------------------------------------------------------------------- /canary-deploy-app-v2-playbook/main.yml: -------------------------------------------------------------------------------- 1 | - hosts: k8s-master 2 | become: yes 3 | user: ubuntu 4 | roles: 5 | - common -------------------------------------------------------------------------------- /k8s-install-playbook/roles/install-helm/tasks/main.yml: -------------------------------------------------------------------------------- 1 | - include: install-helm.yml 2 | - include: install-monit-tools.yml -------------------------------------------------------------------------------- /k8s-install-playbook/hosts: -------------------------------------------------------------------------------- 1 | [k8s-master] 2 | 3 | [k8s-workers] 4 | 5 | [k8s-workers:vars] 6 | K8S_MASTER_NODE_IP= 7 | K8S_API_SERCURE_PORT=6443 8 | -------------------------------------------------------------------------------- /deploy-app-v1-playbook/hosts: -------------------------------------------------------------------------------- 1 | [k8s-master] 2 | 3 | [k8s-workers] 4 | 5 | [k8s-workers:vars] 6 | K8S_MASTER_NODE_IP= 7 | K8S_API_SERCURE_PORT=6443 8 | -------------------------------------------------------------------------------- /deploy-app-v2-playbook/hosts: -------------------------------------------------------------------------------- 1 | [k8s-master] 2 | 3 | [k8s-workers] 4 | 5 | [k8s-workers:vars] 6 | K8S_MASTER_NODE_IP= 7 | K8S_API_SERCURE_PORT=6443 8 | -------------------------------------------------------------------------------- /canary-deploy-app-v2-playbook/hosts: -------------------------------------------------------------------------------- 1 | [k8s-master] 2 | 3 | [k8s-workers] 4 | 5 | [k8s-workers:vars] 6 | K8S_MASTER_NODE_IP= 7 | K8S_API_SERCURE_PORT=6443 8 | -------------------------------------------------------------------------------- /provisioning-playbook/hosts: -------------------------------------------------------------------------------- 1 | [local] 2 | localhost ansible_connection=local ansible_python_interpreter=python gather_facts=False 3 | 4 | [giropopsKubernetes] 5 | -------------------------------------------------------------------------------- /k8s-install-playbook/roles/create-cluster/vars/main.yml: -------------------------------------------------------------------------------- 1 | default_kubernetes_cni_weavenet_manifestUrl: "https://cloud.weave.works/k8s/net?k8s-version=$(kubectl version | base64 | tr -d '\n')" -------------------------------------------------------------------------------- /provisioning-playbook/roles/common/vars/main.yml: -------------------------------------------------------------------------------- 1 | instance_type: t2.medium 2 | security_group: giropopsKubernetes 3 | image: ami-0d2505740b82f7948 4 | keypair: chave 5 | region: us-east-1 6 | count: 3 -------------------------------------------------------------------------------- /k8s-install-playbook/roles/create-cluster/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: enable docker 3 | service: name=docker enabled=yes 4 | 5 | - name: restart docker 6 | service: name=docker state=restarted -------------------------------------------------------------------------------- /k8s-install-playbook/roles/install-helm/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: enable docker 3 | service: name=docker enabled=yes 4 | 5 | - name: restart docker 6 | service: name=docker state=restarted -------------------------------------------------------------------------------- /k8s-install-playbook/roles/install_k8s/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: enable docker 3 | service: name=docker enabled=yes 4 | 5 | - name: restart docker 6 | service: name=docker state=restarted -------------------------------------------------------------------------------- /k8s-install-playbook/roles/join-workers/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: enable docker 3 | service: name=docker enabled=yes 4 | 5 | - name: restart docker 6 | service: name=docker state=restarted -------------------------------------------------------------------------------- /k8s-install-playbook/roles/install-helm/tasks/install-monit-tools.yml: -------------------------------------------------------------------------------- 1 | - name: Install Prometheus 2 | shell: helm install {{ deploy_prometheus }} 3 | register: prometheus_result 4 | 5 | - name: Install Grafana 6 | shell: helm install {{ deploy_grafana }} 7 | register: grafana_result -------------------------------------------------------------------------------- /nginx-prometheus-exporter-v1/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM openresty/openresty:alpine 2 | 3 | LABEL version 1.0.0 4 | COPY nginx.conf /usr/local/openresty/nginx/conf/ 5 | COPY *.vhost /usr/local/openresty/nginx/conf/ 6 | COPY lib/prometheus.lua /usr/local/openresty/luajit/lib 7 | 8 | EXPOSE 80 9 | EXPOSE 32111 10 | 11 | RUN nginx -t -------------------------------------------------------------------------------- /nginx-prometheus-exporter-v2/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM openresty/openresty:alpine 2 | 3 | LABEL version 2.0.0 4 | COPY nginx.conf /usr/local/openresty/nginx/conf/ 5 | COPY *.vhost /usr/local/openresty/nginx/conf/ 6 | COPY lib/prometheus.lua /usr/local/openresty/luajit/lib 7 | 8 | EXPOSE 80 9 | EXPOSE 32111 10 | 11 | RUN nginx -t -------------------------------------------------------------------------------- /k8s-install-playbook/roles/install-helm/vars/main.yml: -------------------------------------------------------------------------------- 1 | deploy_prometheus: "--namespace=monitoring --name=prometheus --version=7.0.0 --set alertmanager.persistentVolume.enabled=false,server.persistentVolume.enabled=false stable/prometheus" 2 | deploy_grafana: "--namespace=monitoring --name=grafana --version=1.12.0 --set=adminUser=admin,adminPassword=admin,service.type=NodePort stable/grafana" -------------------------------------------------------------------------------- /canary-deploy-app-v2-playbook/roles/common/tasks/deploy-app.yml: -------------------------------------------------------------------------------- 1 | - name: Copying yml files to the host 2 | copy: src={{ item.src }} dest={{ item.dest }} 3 | with_items: 4 | - { src: 'app-v2-canary.yml', dest: '/opt/giropops/app-v2-canary.yml' } 5 | register: copying_register 6 | 7 | - name: Deploy Giropops App deployment 8 | shell: kubectl create -f /opt/giropops/app-v2-canary.yml 9 | register: deployment_register -------------------------------------------------------------------------------- /k8s-install-playbook/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - hosts: all 3 | become: yes 4 | user: ubuntu 5 | gather_facts: no 6 | pre_tasks: 7 | - name: 'install python' 8 | raw: 'apt-get -y install python' 9 | roles: 10 | - install_k8s 11 | 12 | - hosts: k8s-master 13 | become: yes 14 | user: ubuntu 15 | roles: 16 | - create-cluster 17 | 18 | - hosts: k8s-workers 19 | become: yes 20 | user: ubuntu 21 | roles: 22 | - join-workers 23 | 24 | 25 | - hosts: k8s-master 26 | become: yes 27 | user: ubuntu 28 | roles: 29 | - install-helm -------------------------------------------------------------------------------- /nginx-prometheus-exporter-v1/nginx.conf: -------------------------------------------------------------------------------- 1 | worker_processes 1; 2 | 3 | events { 4 | worker_connections 1024; 5 | } 6 | 7 | http { 8 | #include mime.types; 9 | default_type text/html; 10 | 11 | sendfile on; 12 | keepalive_timeout 65; 13 | 14 | server { 15 | listen 80; 16 | server_name localhost; 17 | 18 | location / { 19 | content_by_lua_block { 20 | ngx.say("Giropops App - Version 1.0.0") 21 | } 22 | } 23 | } 24 | 25 | include *.vhost; 26 | } 27 | 28 | -------------------------------------------------------------------------------- /nginx-prometheus-exporter-v2/nginx.conf: -------------------------------------------------------------------------------- 1 | worker_processes 1; 2 | 3 | events { 4 | worker_connections 1024; 5 | } 6 | 7 | http { 8 | #include mime.types; 9 | default_type text/html; 10 | 11 | sendfile on; 12 | keepalive_timeout 65; 13 | 14 | server { 15 | listen 80; 16 | server_name localhost; 17 | 18 | location / { 19 | content_by_lua_block { 20 | ngx.say("Giropops App - Version 2.0.0") 21 | } 22 | } 23 | } 24 | 25 | include *.vhost; 26 | } 27 | 28 | -------------------------------------------------------------------------------- /deploy-app-v1-playbook/roles/common/files/service-app.yml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | labels: 5 | app: giropops 6 | run: nginx 7 | track: stable 8 | name: giropops 9 | namespace: default 10 | spec: 11 | externalTrafficPolicy: Cluster 12 | ports: 13 | - nodePort: 32222 14 | name: http 15 | port: 80 16 | protocol: TCP 17 | targetPort: 80 18 | - nodePort: 32111 19 | name: prometheus 20 | port: 32111 21 | protocol: TCP 22 | targetPort: 32111 23 | selector: 24 | app: giropops 25 | sessionAffinity: None 26 | type: NodePort -------------------------------------------------------------------------------- /k8s-install-playbook/roles/install_k8s/tasks/install.yml: -------------------------------------------------------------------------------- 1 | - name: Install Docker 2 | shell: curl -fsSL https://get.docker.com | bash - 3 | 4 | - name: Add Apt signing key 5 | apt_key: 6 | url: https://packages.cloud.google.com/apt/doc/apt-key.gpg 7 | state: present 8 | 9 | - name: Add K8s repository 10 | apt_repository: 11 | repo: deb http://apt.kubernetes.io/ kubernetes-xenial main 12 | state: present 13 | 14 | - name: Install k8s packages 15 | apt: 16 | name: "{{ packages }}" 17 | vars: 18 | packages: 19 | - kubelet 20 | - kubeadm 21 | - kubectl 22 | 23 | -------------------------------------------------------------------------------- /k8s-install-playbook/roles/install_k8s/templates/docker-compose.yml.j2: -------------------------------------------------------------------------------- 1 | version: "3" 2 | services: 3 | mysql: 4 | image: mariadb:10.2.14 5 | ports: 6 | - 3306:3306 7 | volumes: 8 | - data-volume:/var/lib/mysql 9 | - /opt/eriks/config/my.cnf:/etc/mysql/my.cnf 10 | 11 | environment: 12 | - MYSQL_USER={{ mysql_user }} 13 | - MYSQL_PASSWORD={{ mysql_password }} 14 | - MYSQL_ROOT_PASSWORD={{ mysql_root_password }} 15 | - MYSQL_DATABASE=db 16 | 17 | volumes: 18 | data-volume: 19 | driver: local 20 | driver_opts: 21 | type: 'none' 22 | o: 'bind' 23 | device: '/opt/eriks/data' 24 | -------------------------------------------------------------------------------- /k8s-install-playbook/roles/join-workers/templates/docker-compose.yml.j2: -------------------------------------------------------------------------------- 1 | version: "3" 2 | services: 3 | mysql: 4 | image: mariadb:10.2.14 5 | ports: 6 | - 3306:3306 7 | volumes: 8 | - data-volume:/var/lib/mysql 9 | - /opt/eriks/config/my.cnf:/etc/mysql/my.cnf 10 | 11 | environment: 12 | - MYSQL_USER={{ mysql_user }} 13 | - MYSQL_PASSWORD={{ mysql_password }} 14 | - MYSQL_ROOT_PASSWORD={{ mysql_root_password }} 15 | - MYSQL_DATABASE=db 16 | 17 | volumes: 18 | data-volume: 19 | driver: local 20 | driver_opts: 21 | type: 'none' 22 | o: 'bind' 23 | device: '/opt/eriks/data' 24 | -------------------------------------------------------------------------------- /deploy-app-v2-playbook/roles/common/files/app-v1.yml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | kind: Deployment 3 | metadata: 4 | name: giropops-v1 5 | spec: 6 | replicas: 1 7 | template: 8 | metadata: 9 | labels: 10 | app: giropops 11 | version: "1.0.0" 12 | annotations: 13 | prometheus.io/scrape: "true" 14 | prometheus.io/port: "32111" 15 | spec: 16 | containers: 17 | - name: giropops 18 | image: linuxtips/nginx-prometheus-exporter:1.0.0 19 | env: 20 | - name: VERSION 21 | value: "1.0.0" 22 | ports: 23 | - containerPort: 80 24 | - containerPort: 32111 -------------------------------------------------------------------------------- /deploy-app-v1-playbook/roles/common/files/app-v1.yml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | kind: Deployment 3 | metadata: 4 | name: giropops-v1 5 | spec: 6 | replicas: 10 7 | template: 8 | metadata: 9 | labels: 10 | app: giropops 11 | version: "1.0.0" 12 | annotations: 13 | prometheus.io/scrape: "true" 14 | prometheus.io/port: "32111" 15 | spec: 16 | containers: 17 | - name: giropops 18 | image: linuxtips/nginx-prometheus-exporter:1.0.0 19 | env: 20 | - name: VERSION 21 | value: "1.0.0" 22 | ports: 23 | - containerPort: 80 24 | - containerPort: 32111 -------------------------------------------------------------------------------- /canary-deploy-app-v2-playbook/roles/common/files/app-v2-canary.yml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | kind: Deployment 3 | metadata: 4 | name: giropops-v2 5 | spec: 6 | replicas: 1 7 | template: 8 | metadata: 9 | labels: 10 | app: giropops 11 | version: "2.0.0" 12 | annotations: 13 | prometheus.io/scrape: "true" 14 | prometheus.io/port: "32111" 15 | spec: 16 | containers: 17 | - name: giropops 18 | image: linuxtips/nginx-prometheus-exporter:2.0.0 19 | env: 20 | - name: VERSION 21 | value: "2.0.0" 22 | ports: 23 | - containerPort: 80 24 | - containerPort: 32111 -------------------------------------------------------------------------------- /k8s-install-playbook/roles/join-workers/tasks/join-cluster.yml: -------------------------------------------------------------------------------- 1 | - name: 2 | debug: 3 | msg: "[Worker] K8S_TOKEN_HOLDER K8S token is {{ hostvars['K8S_TOKEN_HOLDER']['token'] }}" 4 | 5 | - name: 6 | debug: 7 | msg: "[Worker] K8S_TOKEN_HOLDER K8S Hash is {{ hostvars['K8S_TOKEN_HOLDER']['hash'] }}" 8 | 9 | - name: "Kubeadmn reset node cluster config" 10 | command: > 11 | kubeadm reset --force 12 | register: kubeadm-reset_node 13 | 14 | - name: "Kubeadmn join" 15 | shell: > 16 | kubeadm join --token={{ hostvars['K8S_TOKEN_HOLDER']['token'] }} 17 | --discovery-token-ca-cert-hash sha256:{{ hostvars['K8S_TOKEN_HOLDER']['hash'] }} 18 | {{K8S_MASTER_NODE_IP}}:{{K8S_API_SERCURE_PORT}} -------------------------------------------------------------------------------- /deploy-app-v1-playbook/roles/common/tasks/deploy-app.yml: -------------------------------------------------------------------------------- 1 | - name: Creating Giropops App directory 2 | file: path={{item}} state=directory 3 | with_items: 4 | - /opt/giropops 5 | register: directory_app_register 6 | 7 | - name: Copying yml files to the host 8 | copy: src={{ item.src }} dest={{ item.dest }} 9 | with_items: 10 | - { src: 'app-v1.yml', dest: '/opt/giropops/app-v1.yml' } 11 | - { src: 'service-app.yml', dest: '/opt/giropops/service-app.yml' } 12 | register: copying_register 13 | 14 | - name: Deploy Giropops App deployment 15 | shell: kubectl create -f /opt/giropops/app-v1.yml 16 | register: deployment_register 17 | 18 | - name: Deploy Giropops App service 19 | shell: kubectl create -f /opt/giropops/service-app.yml 20 | register: deployment_register -------------------------------------------------------------------------------- /k8s-install-playbook/roles/install-helm/tasks/install-helm.yml: -------------------------------------------------------------------------------- 1 | - name: Install helm via snap 2 | shell: snap install helm --classic 3 | register: helm_result 4 | 5 | - name: Helm init 6 | shell: helm init 7 | register: helm_init_result 8 | 9 | - name: Create service account to tiller 10 | shell: kubectl create serviceaccount --namespace=kube-system tiller 11 | register: tiller_result 12 | 13 | - name: Create clusterrolebinding for tiller 14 | shell: kubectl create clusterrolebinding tiller-cluster-role --clusterrole=cluster-admin --serviceaccount=kube-system:tiller 15 | register: clusterrolebinding_result 16 | 17 | - name: Apply patch to tiller-deploy 18 | shell: kubectl patch deployments -n kube-system tiller-deploy -p '{"spec":{"template":{"spec":{"serviceAccount":"tiller"}}}}' 19 | register: patch_result 20 | 21 | - name: Waiting tiller pod 22 | pause: 23 | minutes: 2 -------------------------------------------------------------------------------- /deploy-app-v2-playbook/roles/common/files/app-v2.yml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | kind: Deployment 3 | metadata: 4 | name: giropops-v2 5 | spec: 6 | replicas: 10 7 | template: 8 | metadata: 9 | labels: 10 | app: giropops 11 | version: "2.0.0" 12 | annotations: 13 | prometheus.io/scrape: "true" 14 | prometheus.io/port: "32111" 15 | spec: 16 | containers: 17 | - name: giropops 18 | image: linuxtips/nginx-prometheus-exporter:2.0.0 19 | env: 20 | - name: VERSION 21 | value: "2.0.0" 22 | livenessProbe: 23 | httpGet: 24 | path: / 25 | port: 80 26 | scheme: HTTP 27 | readinessProbe: 28 | httpGet: 29 | path: / 30 | port: 80 31 | scheme: HTTP 32 | ports: 33 | - containerPort: 80 34 | - containerPort: 32111 -------------------------------------------------------------------------------- /deploy-app-v2-playbook/roles/common/tasks/deploy-app.yml: -------------------------------------------------------------------------------- 1 | - name: Copying yml files to the host 2 | copy: src={{ item.src }} dest={{ item.dest }} 3 | with_items: 4 | - { src: 'app-v2.yml', dest: '/opt/giropops/app-v2.yml' } 5 | - { src: 'app-v1.yml', dest: '/opt/giropops/app-v1.yml' } 6 | 7 | register: copying_register 8 | 9 | - name: Deploy new version of Giropops App deployment 10 | shell: kubectl apply -f /opt/giropops/app-v2.yml 11 | register: deployment_register 12 | 13 | - name: Scale down old version of Giropops App deployment 14 | shell: kubectl apply -f /opt/giropops/app-v1.yml 15 | register: deployment_register 16 | 17 | - name: The old version of Giropops App deployment will be removed in two minutes 18 | pause: 19 | minutes: 2 20 | 21 | - name: Delete old version of Giropops App deployment 22 | shell: kubectl delete -f /opt/giropops/app-v1.yml 23 | register: deployment_deleted_register -------------------------------------------------------------------------------- /k8s-install-playbook/roles/create-cluster/tasks/init-cluster.yml: -------------------------------------------------------------------------------- 1 | - name: Reset Cluster 2 | command: > 3 | kubeadm reset --force 4 | register: kubeadmin_init 5 | 6 | - name: Initialize Kubernetes master with kubeadm init. 7 | command: > 8 | kubeadm init 9 | register: kubeadmin_init 10 | 11 | - name: Ensure .kube directory exists. 12 | file: 13 | path: ~/.kube 14 | state: directory 15 | 16 | - name: Symlink the kubectl admin.conf to ~/.kube/conf. 17 | file: 18 | src: /etc/kubernetes/admin.conf 19 | dest: ~/.kube/config 20 | state: link 21 | 22 | - name: Configure weavenet networking. 23 | shell: kubectl apply -f {{ default_kubernetes_cni_weavenet_manifestUrl }} 24 | register: weavenet_result 25 | 26 | - name: "Cluster token" 27 | shell: kubeadm token list | cut -d ' ' -f1 | sed -n '2p' 28 | register: K8S_TOKEN 29 | 30 | - name: "CA Hash" 31 | shell: openssl x509 -pubkey -in /etc/kubernetes/pki/ca.crt | openssl rsa -pubin -outform der 2>/dev/null | openssl dgst -sha256 -hex | sed 's/^.* //' 32 | register: K8S_MASTER_CA_HASH 33 | 34 | - name: "Add K8S Token and Hash to dummy host" 35 | add_host: 36 | name: "K8S_TOKEN_HOLDER" 37 | token: "{{ K8S_TOKEN.stdout }}" 38 | hash: "{{ K8S_MASTER_CA_HASH.stdout }}" 39 | 40 | - name: 41 | debug: 42 | msg: "[Master] K8S_TOKEN_HOLDER K8S token is {{ hostvars['K8S_TOKEN_HOLDER']['token'] }}" 43 | 44 | - name: 45 | debug: 46 | msg: "[Master] K8S_TOKEN_HOLDER K8S Hash is {{ hostvars['K8S_TOKEN_HOLDER']['hash'] }}" -------------------------------------------------------------------------------- /provisioning-playbook/roles/common/tasks/provisioning.yml: -------------------------------------------------------------------------------- 1 | - name: Create a security group 2 | local_action: 3 | module: ec2_group 4 | name: "{{ security_group }}" 5 | description: Security Group for webserver Servers 6 | region: "{{ region }}" 7 | rules: 8 | - proto: tcp 9 | from_port: 22 10 | to_port: 22 11 | cidr_ip: 0.0.0.0/0 12 | - proto: tcp 13 | from_port: 32222 14 | to_port: 32222 15 | cidr_ip: 0.0.0.0/0 16 | - proto: tcp 17 | from_port: 32111 18 | to_port: 32111 19 | cidr_ip: 0.0.0.0/0 20 | - proto: tcp 21 | from_port: 32112 22 | to_port: 32112 23 | cidr_ip: 0.0.0.0/0 24 | rules_egress: 25 | - proto: all 26 | cidr_ip: 0.0.0.0/0 27 | register: basic_firewall 28 | 29 | - name: Launch the new EC2 Instances 30 | local_action: ec2 31 | group={{ security_group }} 32 | instance_type={{ instance_type}} 33 | image={{ image }} 34 | wait=true 35 | region={{ region }} 36 | keypair={{ keypair }} 37 | count={{count}} 38 | register: ec2 39 | 40 | - name: add ec2 instances to a temporary inventory 41 | add_host: name={{ item.public_ip }} groups=debijenkorf-new 42 | with_items: "{{ ec2.instances }}" 43 | 44 | - name: Add the newly created EC2 instance into inventory 45 | local_action: lineinfile 46 | dest="./hosts" 47 | regexp={{ item.public_ip }} 48 | insertafter="[giropopsKubernetes]" line={{ item.public_ip }} 49 | with_items: "{{ ec2.instances }}" 50 | 51 | - name: Add the private IP of newly created EC2 instance into inventory 52 | local_action: lineinfile 53 | dest="./hosts" 54 | regexp={{ item.private_ip }} 55 | insertafter="[giropopsKubernetes]" line={{ item.private_ip }} 56 | with_items: "{{ ec2.instances }}" 57 | 58 | - name: Wait for SSH 59 | local_action: wait_for 60 | host={{ item.public_ip }} 61 | port=22 62 | state=started 63 | with_items: "{{ ec2.instances }}" 64 | 65 | - name: Add the newly created EC2 into known_hosts 66 | shell: ssh-keyscan -H {{ item.public_ip }} >> ~/.ssh/known_hosts 67 | with_items: "{{ ec2.instances }}" 68 | 69 | - name: Add tag to Instance 70 | local_action: ec2_tag resource={{ item.id }} region={{ region }} state=present 71 | with_items: "{{ ec2.instances }}" 72 | args: 73 | tags: 74 | Name: giropopsKubernetes 75 | -------------------------------------------------------------------------------- /k8s-install-playbook/roles/create-cluster/files/my.cnf: -------------------------------------------------------------------------------- 1 | # MariaDB database server configuration file. 2 | # 3 | # You can copy this file to one of: 4 | # - "/etc/mysql/my.cnf" to set global options, 5 | # - "~/.my.cnf" to set user-specific options. 6 | # 7 | # One can use all long options that the program supports. 8 | # Run program with --help to get a list of available options and with 9 | # --print-defaults to see which it would actually understand and use. 10 | # 11 | # For explanations see 12 | # http://dev.mysql.com/doc/mysql/en/server-system-variables.html 13 | 14 | # This will be passed to all mysql clients 15 | # It has been reported that passwords should be enclosed with ticks/quotes 16 | # escpecially if they contain "#" chars... 17 | # Remember to edit /etc/mysql/debian.cnf when changing the socket location. 18 | [client] 19 | port = 3306 20 | socket = /var/run/mysqld/mysqld.sock 21 | 22 | # Here is entries for some specific programs 23 | # The following values assume you have at least 32M ram 24 | 25 | # This was formally known as [safe_mysqld]. Both versions are currently parsed. 26 | [mysqld_safe] 27 | socket = /var/run/mysqld/mysqld.sock 28 | nice = 0 29 | 30 | [mysqld] 31 | # 32 | # * Basic Settings 33 | # 34 | #user = mysql 35 | pid-file = /var/run/mysqld/mysqld.pid 36 | socket = /var/run/mysqld/mysqld.sock 37 | port = 3306 38 | basedir = /usr 39 | datadir = /var/lib/mysql 40 | tmpdir = /tmp 41 | lc_messages_dir = /usr/share/mysql 42 | lc_messages = en_US 43 | skip-external-locking 44 | # 45 | # Instead of skip-networking the default is now to listen only on 46 | # localhost which is more compatible and is not less secure. 47 | bind-address = 0.0.0.0 48 | # 49 | # * Fine Tuning 50 | # 51 | max_connections = 100 52 | connect_timeout = 5 53 | wait_timeout = 600 54 | max_allowed_packet = 16M 55 | thread_cache_size = 128 56 | sort_buffer_size = 4M 57 | bulk_insert_buffer_size = 16M 58 | tmp_table_size = 32M 59 | max_heap_table_size = 32M 60 | # 61 | # * MyISAM 62 | # 63 | # This replaces the startup script and checks MyISAM tables if needed 64 | # the first time they are touched. On error, make copy and try a repair. 65 | myisam_recover_options = BACKUP 66 | key_buffer_size = 128M 67 | #open-files-limit = 2000 68 | table_open_cache = 400 69 | myisam_sort_buffer_size = 512M 70 | concurrent_insert = 2 71 | read_buffer_size = 2M 72 | read_rnd_buffer_size = 1M 73 | # 74 | # * Query Cache Configuration 75 | # 76 | # Cache only tiny result sets, so we can fit more in the query cache. 77 | query_cache_limit = 128K 78 | query_cache_size = 64M 79 | # for more write intensive setups, set to DEMAND or OFF 80 | #query_cache_type = DEMAND 81 | # 82 | # * Logging and Replication 83 | # 84 | # Both location gets rotated by the cronjob. 85 | # Be aware that this log type is a performance killer. 86 | # As of 5.1 you can enable the log at runtime! 87 | #general_log_file = /var/log/mysql/mysql.log 88 | #general_log = 1 89 | # 90 | # Error logging goes to syslog due to /etc/mysql/conf.d/mysqld_safe_syslog.cnf. 91 | # 92 | # we do want to know about network errors and such 93 | #log_warnings = 2 94 | # 95 | # Enable the slow query log to see queries with especially long duration 96 | #slow_query_log[={0|1}] 97 | slow_query_log_file = /var/log/mysql/mariadb-slow.log 98 | long_query_time = 10 99 | #log_slow_rate_limit = 1000 100 | #log_slow_verbosity = query_plan 101 | 102 | #log-queries-not-using-indexes 103 | #log_slow_admin_statements 104 | # 105 | # The following can be used as easy to replay backup logs or for replication. 106 | # note: if you are setting up a replication slave, see README.Debian about 107 | # other settings you may need to change. 108 | #server-id = 1 109 | #report_host = master1 110 | #auto_increment_increment = 2 111 | #auto_increment_offset = 1 112 | #log_bin = /var/log/mysql/mariadb-bin 113 | #log_bin_index = /var/log/mysql/mariadb-bin.index 114 | # not fab for performance, but safer 115 | #sync_binlog = 1 116 | expire_logs_days = 10 117 | max_binlog_size = 100M 118 | # slaves 119 | #relay_log = /var/log/mysql/relay-bin 120 | #relay_log_index = /var/log/mysql/relay-bin.index 121 | #relay_log_info_file = /var/log/mysql/relay-bin.info 122 | #log_slave_updates 123 | #read_only 124 | # 125 | # If applications support it, this stricter sql_mode prevents some 126 | # mistakes like inserting invalid dates etc. 127 | #sql_mode = NO_ENGINE_SUBSTITUTION,TRADITIONAL 128 | # 129 | # * InnoDB 130 | # 131 | # InnoDB is enabled by default with a 10MB datafile in /var/lib/mysql/. 132 | # Read the manual for more InnoDB related options. There are many! 133 | default_storage_engine = InnoDB 134 | # you can't just change log file size, requires special procedure 135 | #innodb_log_file_size = 50M 136 | innodb_buffer_pool_size = 256M 137 | innodb_log_buffer_size = 8M 138 | innodb_file_per_table = 1 139 | innodb_open_files = 400 140 | innodb_io_capacity = 400 141 | innodb_flush_method = O_DIRECT 142 | # 143 | # * Security Features 144 | # 145 | # Read the manual, too, if you want chroot! 146 | # chroot = /var/lib/mysql/ 147 | # 148 | # For generating SSL certificates I recommend the OpenSSL GUI "tinyca". 149 | # 150 | # ssl-ca=/etc/mysql/cacert.pem 151 | # ssl-cert=/etc/mysql/server-cert.pem 152 | # ssl-key=/etc/mysql/server-key.pem 153 | 154 | # 155 | # * Galera-related settings 156 | # 157 | [galera] 158 | # Mandatory settings 159 | #wsrep_on=ON 160 | #wsrep_provider= 161 | #wsrep_cluster_address= 162 | #binlog_format=row 163 | #default_storage_engine=InnoDB 164 | #innodb_autoinc_lock_mode=2 165 | # 166 | # Allow server to accept connections on all interfaces. 167 | # 168 | #bind-address=0.0.0.0 169 | # 170 | # Optional setting 171 | #wsrep_slave_threads=1 172 | #innodb_flush_log_at_trx_commit=0 173 | 174 | [mysqldump] 175 | quick 176 | quote-names 177 | max_allowed_packet = 16M 178 | 179 | [mysql] 180 | #no-auto-rehash # faster start of mysql but no tab completion 181 | 182 | [isamchk] 183 | key_buffer = 16M 184 | 185 | # 186 | # * IMPORTANT: Additional settings that can override those from this file! 187 | # The files must end with '.cnf', otherwise they'll be ignored. 188 | # 189 | !includedir /etc/mysql/conf.d/ -------------------------------------------------------------------------------- /k8s-install-playbook/roles/install_k8s/files/my.cnf: -------------------------------------------------------------------------------- 1 | # MariaDB database server configuration file. 2 | # 3 | # You can copy this file to one of: 4 | # - "/etc/mysql/my.cnf" to set global options, 5 | # - "~/.my.cnf" to set user-specific options. 6 | # 7 | # One can use all long options that the program supports. 8 | # Run program with --help to get a list of available options and with 9 | # --print-defaults to see which it would actually understand and use. 10 | # 11 | # For explanations see 12 | # http://dev.mysql.com/doc/mysql/en/server-system-variables.html 13 | 14 | # This will be passed to all mysql clients 15 | # It has been reported that passwords should be enclosed with ticks/quotes 16 | # escpecially if they contain "#" chars... 17 | # Remember to edit /etc/mysql/debian.cnf when changing the socket location. 18 | [client] 19 | port = 3306 20 | socket = /var/run/mysqld/mysqld.sock 21 | 22 | # Here is entries for some specific programs 23 | # The following values assume you have at least 32M ram 24 | 25 | # This was formally known as [safe_mysqld]. Both versions are currently parsed. 26 | [mysqld_safe] 27 | socket = /var/run/mysqld/mysqld.sock 28 | nice = 0 29 | 30 | [mysqld] 31 | # 32 | # * Basic Settings 33 | # 34 | #user = mysql 35 | pid-file = /var/run/mysqld/mysqld.pid 36 | socket = /var/run/mysqld/mysqld.sock 37 | port = 3306 38 | basedir = /usr 39 | datadir = /var/lib/mysql 40 | tmpdir = /tmp 41 | lc_messages_dir = /usr/share/mysql 42 | lc_messages = en_US 43 | skip-external-locking 44 | # 45 | # Instead of skip-networking the default is now to listen only on 46 | # localhost which is more compatible and is not less secure. 47 | bind-address = 0.0.0.0 48 | # 49 | # * Fine Tuning 50 | # 51 | max_connections = 100 52 | connect_timeout = 5 53 | wait_timeout = 600 54 | max_allowed_packet = 16M 55 | thread_cache_size = 128 56 | sort_buffer_size = 4M 57 | bulk_insert_buffer_size = 16M 58 | tmp_table_size = 32M 59 | max_heap_table_size = 32M 60 | # 61 | # * MyISAM 62 | # 63 | # This replaces the startup script and checks MyISAM tables if needed 64 | # the first time they are touched. On error, make copy and try a repair. 65 | myisam_recover_options = BACKUP 66 | key_buffer_size = 128M 67 | #open-files-limit = 2000 68 | table_open_cache = 400 69 | myisam_sort_buffer_size = 512M 70 | concurrent_insert = 2 71 | read_buffer_size = 2M 72 | read_rnd_buffer_size = 1M 73 | # 74 | # * Query Cache Configuration 75 | # 76 | # Cache only tiny result sets, so we can fit more in the query cache. 77 | query_cache_limit = 128K 78 | query_cache_size = 64M 79 | # for more write intensive setups, set to DEMAND or OFF 80 | #query_cache_type = DEMAND 81 | # 82 | # * Logging and Replication 83 | # 84 | # Both location gets rotated by the cronjob. 85 | # Be aware that this log type is a performance killer. 86 | # As of 5.1 you can enable the log at runtime! 87 | #general_log_file = /var/log/mysql/mysql.log 88 | #general_log = 1 89 | # 90 | # Error logging goes to syslog due to /etc/mysql/conf.d/mysqld_safe_syslog.cnf. 91 | # 92 | # we do want to know about network errors and such 93 | #log_warnings = 2 94 | # 95 | # Enable the slow query log to see queries with especially long duration 96 | #slow_query_log[={0|1}] 97 | slow_query_log_file = /var/log/mysql/mariadb-slow.log 98 | long_query_time = 10 99 | #log_slow_rate_limit = 1000 100 | #log_slow_verbosity = query_plan 101 | 102 | #log-queries-not-using-indexes 103 | #log_slow_admin_statements 104 | # 105 | # The following can be used as easy to replay backup logs or for replication. 106 | # note: if you are setting up a replication slave, see README.Debian about 107 | # other settings you may need to change. 108 | #server-id = 1 109 | #report_host = master1 110 | #auto_increment_increment = 2 111 | #auto_increment_offset = 1 112 | #log_bin = /var/log/mysql/mariadb-bin 113 | #log_bin_index = /var/log/mysql/mariadb-bin.index 114 | # not fab for performance, but safer 115 | #sync_binlog = 1 116 | expire_logs_days = 10 117 | max_binlog_size = 100M 118 | # slaves 119 | #relay_log = /var/log/mysql/relay-bin 120 | #relay_log_index = /var/log/mysql/relay-bin.index 121 | #relay_log_info_file = /var/log/mysql/relay-bin.info 122 | #log_slave_updates 123 | #read_only 124 | # 125 | # If applications support it, this stricter sql_mode prevents some 126 | # mistakes like inserting invalid dates etc. 127 | #sql_mode = NO_ENGINE_SUBSTITUTION,TRADITIONAL 128 | # 129 | # * InnoDB 130 | # 131 | # InnoDB is enabled by default with a 10MB datafile in /var/lib/mysql/. 132 | # Read the manual for more InnoDB related options. There are many! 133 | default_storage_engine = InnoDB 134 | # you can't just change log file size, requires special procedure 135 | #innodb_log_file_size = 50M 136 | innodb_buffer_pool_size = 256M 137 | innodb_log_buffer_size = 8M 138 | innodb_file_per_table = 1 139 | innodb_open_files = 400 140 | innodb_io_capacity = 400 141 | innodb_flush_method = O_DIRECT 142 | # 143 | # * Security Features 144 | # 145 | # Read the manual, too, if you want chroot! 146 | # chroot = /var/lib/mysql/ 147 | # 148 | # For generating SSL certificates I recommend the OpenSSL GUI "tinyca". 149 | # 150 | # ssl-ca=/etc/mysql/cacert.pem 151 | # ssl-cert=/etc/mysql/server-cert.pem 152 | # ssl-key=/etc/mysql/server-key.pem 153 | 154 | # 155 | # * Galera-related settings 156 | # 157 | [galera] 158 | # Mandatory settings 159 | #wsrep_on=ON 160 | #wsrep_provider= 161 | #wsrep_cluster_address= 162 | #binlog_format=row 163 | #default_storage_engine=InnoDB 164 | #innodb_autoinc_lock_mode=2 165 | # 166 | # Allow server to accept connections on all interfaces. 167 | # 168 | #bind-address=0.0.0.0 169 | # 170 | # Optional setting 171 | #wsrep_slave_threads=1 172 | #innodb_flush_log_at_trx_commit=0 173 | 174 | [mysqldump] 175 | quick 176 | quote-names 177 | max_allowed_packet = 16M 178 | 179 | [mysql] 180 | #no-auto-rehash # faster start of mysql but no tab completion 181 | 182 | [isamchk] 183 | key_buffer = 16M 184 | 185 | # 186 | # * IMPORTANT: Additional settings that can override those from this file! 187 | # The files must end with '.cnf', otherwise they'll be ignored. 188 | # 189 | !includedir /etc/mysql/conf.d/ -------------------------------------------------------------------------------- /k8s-install-playbook/roles/join-workers/files/my.cnf: -------------------------------------------------------------------------------- 1 | # MariaDB database server configuration file. 2 | # 3 | # You can copy this file to one of: 4 | # - "/etc/mysql/my.cnf" to set global options, 5 | # - "~/.my.cnf" to set user-specific options. 6 | # 7 | # One can use all long options that the program supports. 8 | # Run program with --help to get a list of available options and with 9 | # --print-defaults to see which it would actually understand and use. 10 | # 11 | # For explanations see 12 | # http://dev.mysql.com/doc/mysql/en/server-system-variables.html 13 | 14 | # This will be passed to all mysql clients 15 | # It has been reported that passwords should be enclosed with ticks/quotes 16 | # escpecially if they contain "#" chars... 17 | # Remember to edit /etc/mysql/debian.cnf when changing the socket location. 18 | [client] 19 | port = 3306 20 | socket = /var/run/mysqld/mysqld.sock 21 | 22 | # Here is entries for some specific programs 23 | # The following values assume you have at least 32M ram 24 | 25 | # This was formally known as [safe_mysqld]. Both versions are currently parsed. 26 | [mysqld_safe] 27 | socket = /var/run/mysqld/mysqld.sock 28 | nice = 0 29 | 30 | [mysqld] 31 | # 32 | # * Basic Settings 33 | # 34 | #user = mysql 35 | pid-file = /var/run/mysqld/mysqld.pid 36 | socket = /var/run/mysqld/mysqld.sock 37 | port = 3306 38 | basedir = /usr 39 | datadir = /var/lib/mysql 40 | tmpdir = /tmp 41 | lc_messages_dir = /usr/share/mysql 42 | lc_messages = en_US 43 | skip-external-locking 44 | # 45 | # Instead of skip-networking the default is now to listen only on 46 | # localhost which is more compatible and is not less secure. 47 | bind-address = 0.0.0.0 48 | # 49 | # * Fine Tuning 50 | # 51 | max_connections = 100 52 | connect_timeout = 5 53 | wait_timeout = 600 54 | max_allowed_packet = 16M 55 | thread_cache_size = 128 56 | sort_buffer_size = 4M 57 | bulk_insert_buffer_size = 16M 58 | tmp_table_size = 32M 59 | max_heap_table_size = 32M 60 | # 61 | # * MyISAM 62 | # 63 | # This replaces the startup script and checks MyISAM tables if needed 64 | # the first time they are touched. On error, make copy and try a repair. 65 | myisam_recover_options = BACKUP 66 | key_buffer_size = 128M 67 | #open-files-limit = 2000 68 | table_open_cache = 400 69 | myisam_sort_buffer_size = 512M 70 | concurrent_insert = 2 71 | read_buffer_size = 2M 72 | read_rnd_buffer_size = 1M 73 | # 74 | # * Query Cache Configuration 75 | # 76 | # Cache only tiny result sets, so we can fit more in the query cache. 77 | query_cache_limit = 128K 78 | query_cache_size = 64M 79 | # for more write intensive setups, set to DEMAND or OFF 80 | #query_cache_type = DEMAND 81 | # 82 | # * Logging and Replication 83 | # 84 | # Both location gets rotated by the cronjob. 85 | # Be aware that this log type is a performance killer. 86 | # As of 5.1 you can enable the log at runtime! 87 | #general_log_file = /var/log/mysql/mysql.log 88 | #general_log = 1 89 | # 90 | # Error logging goes to syslog due to /etc/mysql/conf.d/mysqld_safe_syslog.cnf. 91 | # 92 | # we do want to know about network errors and such 93 | #log_warnings = 2 94 | # 95 | # Enable the slow query log to see queries with especially long duration 96 | #slow_query_log[={0|1}] 97 | slow_query_log_file = /var/log/mysql/mariadb-slow.log 98 | long_query_time = 10 99 | #log_slow_rate_limit = 1000 100 | #log_slow_verbosity = query_plan 101 | 102 | #log-queries-not-using-indexes 103 | #log_slow_admin_statements 104 | # 105 | # The following can be used as easy to replay backup logs or for replication. 106 | # note: if you are setting up a replication slave, see README.Debian about 107 | # other settings you may need to change. 108 | #server-id = 1 109 | #report_host = master1 110 | #auto_increment_increment = 2 111 | #auto_increment_offset = 1 112 | #log_bin = /var/log/mysql/mariadb-bin 113 | #log_bin_index = /var/log/mysql/mariadb-bin.index 114 | # not fab for performance, but safer 115 | #sync_binlog = 1 116 | expire_logs_days = 10 117 | max_binlog_size = 100M 118 | # slaves 119 | #relay_log = /var/log/mysql/relay-bin 120 | #relay_log_index = /var/log/mysql/relay-bin.index 121 | #relay_log_info_file = /var/log/mysql/relay-bin.info 122 | #log_slave_updates 123 | #read_only 124 | # 125 | # If applications support it, this stricter sql_mode prevents some 126 | # mistakes like inserting invalid dates etc. 127 | #sql_mode = NO_ENGINE_SUBSTITUTION,TRADITIONAL 128 | # 129 | # * InnoDB 130 | # 131 | # InnoDB is enabled by default with a 10MB datafile in /var/lib/mysql/. 132 | # Read the manual for more InnoDB related options. There are many! 133 | default_storage_engine = InnoDB 134 | # you can't just change log file size, requires special procedure 135 | #innodb_log_file_size = 50M 136 | innodb_buffer_pool_size = 256M 137 | innodb_log_buffer_size = 8M 138 | innodb_file_per_table = 1 139 | innodb_open_files = 400 140 | innodb_io_capacity = 400 141 | innodb_flush_method = O_DIRECT 142 | # 143 | # * Security Features 144 | # 145 | # Read the manual, too, if you want chroot! 146 | # chroot = /var/lib/mysql/ 147 | # 148 | # For generating SSL certificates I recommend the OpenSSL GUI "tinyca". 149 | # 150 | # ssl-ca=/etc/mysql/cacert.pem 151 | # ssl-cert=/etc/mysql/server-cert.pem 152 | # ssl-key=/etc/mysql/server-key.pem 153 | 154 | # 155 | # * Galera-related settings 156 | # 157 | [galera] 158 | # Mandatory settings 159 | #wsrep_on=ON 160 | #wsrep_provider= 161 | #wsrep_cluster_address= 162 | #binlog_format=row 163 | #default_storage_engine=InnoDB 164 | #innodb_autoinc_lock_mode=2 165 | # 166 | # Allow server to accept connections on all interfaces. 167 | # 168 | #bind-address=0.0.0.0 169 | # 170 | # Optional setting 171 | #wsrep_slave_threads=1 172 | #innodb_flush_log_at_trx_commit=0 173 | 174 | [mysqldump] 175 | quick 176 | quote-names 177 | max_allowed_packet = 16M 178 | 179 | [mysql] 180 | #no-auto-rehash # faster start of mysql but no tab completion 181 | 182 | [isamchk] 183 | key_buffer = 16M 184 | 185 | # 186 | # * IMPORTANT: Additional settings that can override those from this file! 187 | # The files must end with '.cnf', otherwise they'll be ignored. 188 | # 189 | !includedir /etc/mysql/conf.d/ -------------------------------------------------------------------------------- /nginx-prometheus-exporter-v1/metrics.vhost: -------------------------------------------------------------------------------- 1 | lua_shared_dict prometheus_metrics 10M; 2 | lua_package_path '/usr/local/openresty/luajit/lib/?.lua;;'; 3 | 4 | init_by_lua_block { 5 | prometheus = require("prometheus").init("prometheus_metrics") 6 | 7 | http_requests = prometheus:counter( 8 | "nginx_http_requests", "Number of HTTP requests", {"host", "status"}) 9 | http_request_time = prometheus:histogram( 10 | "nginx_http_request_time", "HTTP request time", {"host"}) 11 | http_request_bytes_received = prometheus:counter( 12 | "nginx_http_request_bytes_received", "Number of HTTP request bytes received", {"host"}) 13 | http_request_bytes_sent = prometheus:counter( 14 | "nginx_http_request_bytes_sent", "Number of HTTP request bytes sent", {"host"}) 15 | http_connections = prometheus:gauge( 16 | "nginx_http_connections", "Number of HTTP connections", {"state"}) 17 | http_upstream_cache_status = prometheus:counter( 18 | "nginx_http_upstream_cache_status", "Number of HTTP upstream cache status", {"host", "status"}) 19 | http_upstream_requests = prometheus:counter( 20 | "nginx_http_upstream_requests", "Number of HTTP upstream requests", {"addr", "status"}) 21 | http_upstream_response_time = prometheus:histogram( 22 | "nginx_http_upstream_response_time", "HTTP upstream response time", {"addr"}) 23 | http_upstream_header_time = prometheus:histogram( 24 | "nginx_http_upstream_header_time", "HTTP upstream header time", {"addr"}) 25 | http_upstream_bytes_received = prometheus:counter( 26 | "nginx_http_upstream_bytes_received", "Number of HTTP upstream bytes received", {"addr"}) 27 | http_upstream_bytes_sent = prometheus:counter( 28 | "nginx_http_upstream_bytes_sent", "Number of HTTP upstream bytes sent", {"addr"}) 29 | http_upstream_connect_time = prometheus:histogram( 30 | "nginx_http_upstream_connect_time", "HTTP upstream connect time", {"addr"}) 31 | http_upstream_first_byte_time = prometheus:histogram( 32 | "nginx_http_upstream_first_byte_time", "HTTP upstream first byte time", {"addr"}) 33 | http_upstream_session_time = prometheus:histogram( 34 | "nginx_http_upstream_session_time", "HTTP upstream session time", {"addr"}) 35 | 36 | } 37 | 38 | log_by_lua_block { 39 | local function split(str) 40 | local array = {} 41 | for mem in string.gmatch(str, '([^, ]+)') do 42 | table.insert(array, mem) 43 | end 44 | return array 45 | end 46 | 47 | local function getWithIndex(str, idx) 48 | if str == nil then 49 | return nil 50 | end 51 | 52 | return split(str)[idx] 53 | end 54 | 55 | local host = ngx.var.host 56 | local status = ngx.var.status 57 | 58 | http_requests:inc(1, {host, status}) 59 | http_request_time:observe(ngx.now() - ngx.req.start_time(), {host}) 60 | 61 | http_request_bytes_sent:inc(tonumber(ngx.var.bytes_sent), {host}) 62 | if ngx.var.bytes_received ~= nil then 63 | http_request_bytes_received:inc(tonumber(ngx.var.bytes_received), {host}) 64 | end 65 | 66 | local upstream_cache_status = ngx.var.upstream_cache_status 67 | if upstream_cache_status ~= nil then 68 | http_upstream_cache_status:inc(1, {host, upstream_cache_status}) 69 | end 70 | 71 | local upstream_addr = ngx.var.upstream_addr 72 | if upstream_addr ~= nil then 73 | local addrs = split(upstream_addr) 74 | 75 | local upstream_status = ngx.var.upstream_status 76 | local upstream_response_time = ngx.var.upstream_response_time 77 | local upstream_connect_time = ngx.var.upstream_connect_time 78 | local upstream_first_byte_time = ngx.var.upstream_first_byte_time 79 | local upstream_header_time = ngx.var.upstream_header_time 80 | local upstream_session_time = ngx.var.upstream_session_time 81 | local upstream_bytes_received = ngx.var.upstream_bytes_received 82 | local upstream_bytes_sent = ngx.var.upstream_bytes_sent 83 | 84 | -- compatible for nginx commas format 85 | for idx, addr in ipairs(addrs) do 86 | if table.getn(addrs) > 1 then 87 | upstream_status = getWithIndex(ngx.var.upstream_status, idx) 88 | upstream_response_time = getWithIndex(ngx.var.upstream_response_time, idx) 89 | upstream_connect_time = getWithIndex(ngx.var.upstream_connect_time, idx) 90 | upstream_first_byte_time = getWithIndex(ngx.var.upstream_first_byte_time, idx) 91 | upstream_header_time = getWithIndex(ngx.var.upstream_header_time, idx) 92 | upstream_session_time = getWithIndex(ngx.var.upstream_session_time, idx) 93 | upstream_bytes_received = getWithIndex(ngx.var.upstream_bytes_received, idx) 94 | upstream_bytes_sent = getWithIndex(ngx.var.upstream_bytes_sent, idx) 95 | end 96 | 97 | http_upstream_requests:inc(1, {addr, upstream_status}) 98 | http_upstream_response_time:observe(tonumber(upstream_response_time), {addr}) 99 | http_upstream_header_time:observe(tonumber(upstream_header_time), {addr}) 100 | 101 | -- ngx.config.nginx_version >= 1011004 102 | if upstream_first_byte_time ~= nil then 103 | http_upstream_first_byte_time:observe(tonumber(upstream_first_byte_time), {addr}) 104 | end 105 | if upstream_connect_time ~= nil then 106 | http_upstream_connect_time:observe(tonumber(upstream_connect_time), {addr}) 107 | end 108 | if upstream_session_time ~= nil then 109 | http_upstream_session_time:observe(tonumber(upstream_session_time), {addr}) 110 | end 111 | if upstream_bytes_received ~= nil then 112 | http_upstream_bytes_received:inc(tonumber(upstream_bytes_received), {addr}) 113 | end 114 | if upstream_bytes_sent ~= nil then 115 | http_upstream_bytes_sent:inc(tonumber(upstream_bytes_sent), {addr}) 116 | end 117 | end 118 | end 119 | } 120 | 121 | server { 122 | listen 32111; 123 | #allow 192.168.0.0/16; 124 | #deny all; 125 | 126 | location /metrics { 127 | content_by_lua_block { 128 | if ngx.var.connections_active ~= nil then 129 | http_connections:set(ngx.var.connections_active, {"active"}) 130 | http_connections:set(ngx.var.connections_reading, {"reading"}) 131 | http_connections:set(ngx.var.connections_waiting, {"waiting"}) 132 | http_connections:set(ngx.var.connections_writing, {"writing"}) 133 | end 134 | prometheus:collect() 135 | } 136 | } 137 | } 138 | -------------------------------------------------------------------------------- /nginx-prometheus-exporter-v2/metrics.vhost: -------------------------------------------------------------------------------- 1 | lua_shared_dict prometheus_metrics 10M; 2 | lua_package_path '/usr/local/openresty/luajit/lib/?.lua;;'; 3 | 4 | init_by_lua_block { 5 | prometheus = require("prometheus").init("prometheus_metrics") 6 | 7 | http_requests = prometheus:counter( 8 | "nginx_http_requests", "Number of HTTP requests", {"host", "status"}) 9 | http_request_time = prometheus:histogram( 10 | "nginx_http_request_time", "HTTP request time", {"host"}) 11 | http_request_bytes_received = prometheus:counter( 12 | "nginx_http_request_bytes_received", "Number of HTTP request bytes received", {"host"}) 13 | http_request_bytes_sent = prometheus:counter( 14 | "nginx_http_request_bytes_sent", "Number of HTTP request bytes sent", {"host"}) 15 | http_connections = prometheus:gauge( 16 | "nginx_http_connections", "Number of HTTP connections", {"state"}) 17 | http_upstream_cache_status = prometheus:counter( 18 | "nginx_http_upstream_cache_status", "Number of HTTP upstream cache status", {"host", "status"}) 19 | http_upstream_requests = prometheus:counter( 20 | "nginx_http_upstream_requests", "Number of HTTP upstream requests", {"addr", "status"}) 21 | http_upstream_response_time = prometheus:histogram( 22 | "nginx_http_upstream_response_time", "HTTP upstream response time", {"addr"}) 23 | http_upstream_header_time = prometheus:histogram( 24 | "nginx_http_upstream_header_time", "HTTP upstream header time", {"addr"}) 25 | http_upstream_bytes_received = prometheus:counter( 26 | "nginx_http_upstream_bytes_received", "Number of HTTP upstream bytes received", {"addr"}) 27 | http_upstream_bytes_sent = prometheus:counter( 28 | "nginx_http_upstream_bytes_sent", "Number of HTTP upstream bytes sent", {"addr"}) 29 | http_upstream_connect_time = prometheus:histogram( 30 | "nginx_http_upstream_connect_time", "HTTP upstream connect time", {"addr"}) 31 | http_upstream_first_byte_time = prometheus:histogram( 32 | "nginx_http_upstream_first_byte_time", "HTTP upstream first byte time", {"addr"}) 33 | http_upstream_session_time = prometheus:histogram( 34 | "nginx_http_upstream_session_time", "HTTP upstream session time", {"addr"}) 35 | 36 | } 37 | 38 | log_by_lua_block { 39 | local function split(str) 40 | local array = {} 41 | for mem in string.gmatch(str, '([^, ]+)') do 42 | table.insert(array, mem) 43 | end 44 | return array 45 | end 46 | 47 | local function getWithIndex(str, idx) 48 | if str == nil then 49 | return nil 50 | end 51 | 52 | return split(str)[idx] 53 | end 54 | 55 | local host = ngx.var.host 56 | local status = ngx.var.status 57 | 58 | http_requests:inc(1, {host, status}) 59 | http_request_time:observe(ngx.now() - ngx.req.start_time(), {host}) 60 | 61 | http_request_bytes_sent:inc(tonumber(ngx.var.bytes_sent), {host}) 62 | if ngx.var.bytes_received ~= nil then 63 | http_request_bytes_received:inc(tonumber(ngx.var.bytes_received), {host}) 64 | end 65 | 66 | local upstream_cache_status = ngx.var.upstream_cache_status 67 | if upstream_cache_status ~= nil then 68 | http_upstream_cache_status:inc(1, {host, upstream_cache_status}) 69 | end 70 | 71 | local upstream_addr = ngx.var.upstream_addr 72 | if upstream_addr ~= nil then 73 | local addrs = split(upstream_addr) 74 | 75 | local upstream_status = ngx.var.upstream_status 76 | local upstream_response_time = ngx.var.upstream_response_time 77 | local upstream_connect_time = ngx.var.upstream_connect_time 78 | local upstream_first_byte_time = ngx.var.upstream_first_byte_time 79 | local upstream_header_time = ngx.var.upstream_header_time 80 | local upstream_session_time = ngx.var.upstream_session_time 81 | local upstream_bytes_received = ngx.var.upstream_bytes_received 82 | local upstream_bytes_sent = ngx.var.upstream_bytes_sent 83 | 84 | -- compatible for nginx commas format 85 | for idx, addr in ipairs(addrs) do 86 | if table.getn(addrs) > 1 then 87 | upstream_status = getWithIndex(ngx.var.upstream_status, idx) 88 | upstream_response_time = getWithIndex(ngx.var.upstream_response_time, idx) 89 | upstream_connect_time = getWithIndex(ngx.var.upstream_connect_time, idx) 90 | upstream_first_byte_time = getWithIndex(ngx.var.upstream_first_byte_time, idx) 91 | upstream_header_time = getWithIndex(ngx.var.upstream_header_time, idx) 92 | upstream_session_time = getWithIndex(ngx.var.upstream_session_time, idx) 93 | upstream_bytes_received = getWithIndex(ngx.var.upstream_bytes_received, idx) 94 | upstream_bytes_sent = getWithIndex(ngx.var.upstream_bytes_sent, idx) 95 | end 96 | 97 | http_upstream_requests:inc(1, {addr, upstream_status}) 98 | http_upstream_response_time:observe(tonumber(upstream_response_time), {addr}) 99 | http_upstream_header_time:observe(tonumber(upstream_header_time), {addr}) 100 | 101 | -- ngx.config.nginx_version >= 1011004 102 | if upstream_first_byte_time ~= nil then 103 | http_upstream_first_byte_time:observe(tonumber(upstream_first_byte_time), {addr}) 104 | end 105 | if upstream_connect_time ~= nil then 106 | http_upstream_connect_time:observe(tonumber(upstream_connect_time), {addr}) 107 | end 108 | if upstream_session_time ~= nil then 109 | http_upstream_session_time:observe(tonumber(upstream_session_time), {addr}) 110 | end 111 | if upstream_bytes_received ~= nil then 112 | http_upstream_bytes_received:inc(tonumber(upstream_bytes_received), {addr}) 113 | end 114 | if upstream_bytes_sent ~= nil then 115 | http_upstream_bytes_sent:inc(tonumber(upstream_bytes_sent), {addr}) 116 | end 117 | end 118 | end 119 | } 120 | 121 | server { 122 | listen 32111; 123 | #allow 192.168.0.0/16; 124 | #deny all; 125 | 126 | location /metrics { 127 | content_by_lua_block { 128 | if ngx.var.connections_active ~= nil then 129 | http_connections:set(ngx.var.connections_active, {"active"}) 130 | http_connections:set(ngx.var.connections_reading, {"reading"}) 131 | http_connections:set(ngx.var.connections_waiting, {"waiting"}) 132 | http_connections:set(ngx.var.connections_writing, {"writing"}) 133 | end 134 | prometheus:collect() 135 | } 136 | } 137 | } 138 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Kubernetes Canary Deployment Example 2 | 3 | Only to show how we can use canary deployment using only default components of Kubernetes. (It's a demo, if you think a better solution use Istio. I'll do a demo-app using Istio soon) 4 | 5 | Using Ansible + AWS + Docker + Kubernetes + Helm + Grafana + Prometheus and one demo-app running OpenResty + Nginx with LUA script to expose metrics to Prometheus. 6 | 7 | 8 | [![HOWTO](http://img.youtube.com/vi/CTvsdWZrAW0/0.jpg)](https://youtu.be/CTvsdWZrAW0 "Kubernetes Canary Deployment Example") 9 | 10 | https://www.youtube.com/watch?v=CTvsdWZrAW0 11 | 12 | These playbooks will do: 13 | - Install 03 Ubuntu instances EC2 14 | - Install Docker all nodes 15 | - Install kubeadm, kubectl and kubelet all nodes 16 | - Create a Kubernetes cluster with 01 master and 02 workers 17 | - Install helm 18 | - Install Prometheus 19 | - Install Grafana 20 | - Create two images (Version 1.0.0 and 2.0.0) to run OpenResty + Nginx exposing the metrics to Prometheus 21 | - Deploy in the k8s the app version 1.0.0 with ten replicas 22 | - Deploy a canary deployment of app version 2.0.0 with 01 replicas (represent 10% of my service) 23 | - Deploy the new version and remove the old one 24 | 25 | Using canary deployment, you can deploy a new version of your app without downtime. For example, you can implement a new version of your app and split the request between both versions. You can redirect your requests where 10% go to the new version, and 90% go to the current version. So, you can check the stability of the latest release in the production environment, and after that, you can determine if you go to the new version in production or not. 26 | 27 | Look an example of a dashboard with all requests to the app during deploy of the new version. 28 | 29 | ![Canary Deployment](images/dash.png) 30 | 31 | 32 | ### Prerequisites 33 | 34 | You need to install Ansible in your computer and have an account into AWS. If you need to run it in another cloud your need to change the provisioning playbook. 35 | 36 | To create the instance is needed that you set two environment variables, AWS_SECRET_ACCESS_KEY and AWS_ACCESS_KEY_ID, with your AWS account info. 37 | 38 | ``` 39 | export AWS_ACCESS_KEY_ID="SHDJSJHDJBNTTS" 40 | export AWS_SECRET_ACCESS_KEY="hSs8s8282kkdbJzUdddd/ss/o+ser" 41 | ``` 42 | 43 | Install PIP: 44 | 45 | ``` 46 | curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py 47 | python get-pip.py 48 | ``` 49 | 50 | Install boto3 and ansible: 51 | 52 | ``` 53 | pip install ansible 54 | pip install boto3 55 | ``` 56 | 57 | You need also have a key pair file, for example my-key-pair.pem, that is used to connect into your AWS instances. 58 | 59 | ### Installing 60 | 61 | After you create the variables, you need to clone this repo. 62 | 63 | ``` 64 | git clone 65 | ``` 66 | 67 | Access the playbook directory: 68 | 69 | ``` 70 | cd k8s-canary-deploy-example/provisioning-playbook/ 71 | ``` 72 | 73 | Open the file roles/common/vars/main.yml, and edit any option that you like, for example, the name of your key pair file from AWS. 74 | 75 | ``` 76 | instance_type: t2.medium #type of instance 77 | security_group: giropops-cluster #name of security group that will be created 78 | image: ami-0d2505740b82f7948 #ami Ubuntu 18.04 TLS official 79 | keypair: my-key-pair.pem #your key pair file 80 | region: us-east-1 # region that will be created the instances 81 | count: 3 #number of instances 82 | ``` 83 | 84 | Load your key pair "my-key-pair.pem" that you specified in the in the var file using ssh-keyscan. 85 | 86 | ``` 87 | ssh-add path/of/my-key-pair.pem 88 | ``` 89 | 90 | Execute the playbook: 91 | 92 | ``` 93 | ansible-playbook -i hosts main.yml 94 | ``` 95 | 96 | When the playbook finishes, you can check in hosts file the IP of new instances: 97 | 98 | ``` 99 | cat hosts 100 | [local] 101 | localhost ansible_connection=local ansible_python_interpreter=python gather_facts=False 102 | 103 | [giropops] 104 | IP_EXTERNAL-NODE1 105 | IP_EXTERNAL-NODE2 106 | IP_EXTERNAL-NODE3 107 | IP_INTERNAL-NODE1 108 | IP_INTERNAL-NODE2 109 | IP_INTERNAL-NODE3 110 | 111 | ``` 112 | Great! You have 03 new instances created in your AWS account. 113 | Now, you need to get the IPs and fill the hosts inventory file that exists inside the all other playbooks, like the example below: 114 | 115 | ``` 116 | cat hosts 117 | [k8s-master] 118 | IP_EXTERNAL-NODE1 119 | 120 | [k8s-workers] 121 | IP_EXTERNAL-NODE2 122 | IP_EXTERNAL-NODE3 123 | 124 | [k8s-workers:vars] 125 | K8S_MASTER_NODE_IP=IP_INTERNAL-NODE1 126 | K8S_API_SERCURE_PORT=6443 127 | ``` 128 | 129 | After that, you need to go to the k8s-install-playbook directory: 130 | 131 | ``` 132 | cd k8s-canary-deploy-example/k8s-install-playbook/ 133 | ``` 134 | 135 | Execute the playbook: 136 | 137 | ``` 138 | ansible-playbook -i hosts main.yml 139 | ``` 140 | 141 | Now you have a Kubernetes cluster with 03 nodes and Helm, Prometheus and Grafana installed. 142 | You can check the Grafana port that is exposed in the master node. 143 | 144 | ``` 145 | kubectl get services -n monitoring 146 | ``` 147 | 148 | To access grafana, use you IP_EXTERNAL-NODE:NODEPORT in your browser. 149 | 150 | ``` 151 | user: admin 152 | password: admin 153 | ``` 154 | 155 | Create a new data source to get metrics from Prometheus 156 | 157 | ``` 158 | Name: Prometheus Server 159 | Type: Prometheus 160 | Url: http://IP_INTERNAL-NODE1:9090 161 | Access: Server 162 | ``` 163 | 164 | Save and Test! 165 | 166 | Now, let's go to create a dashboard. 167 | 168 | ``` 169 | Type: Graph 170 | Query sum(rate(nginx_http_requests{app="giropops"}[5m])) by (version) 171 | Legend : {{version}} 172 | Draw Modes: Bars 173 | Stack 174 | Percent 175 | ``` 176 | Save! 177 | 178 | Now, in another terminal execute the command to simulate request the app and let it run to generate requests to us visualize in the Grafana dashboard : 179 | 180 | ``` 181 | while true; do curl http://54.162.92.118:32222/ ; done 182 | ``` 183 | 184 | 185 | Let's deploy the app version 1.0.0: 186 | 187 | ``` 188 | cd k8s-canary-deploy-example/deploy-app-v1-playbook/ 189 | ``` 190 | 191 | Execute the playbook: 192 | 193 | ``` 194 | ansible-playbook -i hosts main.yml 195 | ``` 196 | 197 | Now, you can see the output of curl that you start to receive a message: 198 | 199 | ``` 200 | Giropops App - Version 1.0.0 201 | ``` 202 | 203 | Let's keep the app version 1.0.0 running for 15 minutes, only collect data enough to our Grafana dashboard, like this: 204 | 205 | ![Deploy app version 1.0.0](images/dash-1.png) 206 | 207 | 208 | Now, let's deploy the canary deployment of app version 2.0.0. In this example, we have ten replicas running the version 1.0.0, so, we need to deploy the canary deployment that will represent around 10% of the number of replicas of the app. Now we will have ten replicas running the version 1.0.0 e only one replica running 2.0.0. 209 | In other words, we will have 90% of the requests to 1.0.0 and 10% of all requests arriving in the version 2.0.0. 210 | 211 | Let's deploy the canary deployment: 212 | 213 | ``` 214 | cd k8s-canary-deploy-example/canary-deploy-app-v2-playbook/ 215 | ``` 216 | 217 | Execute the playbook: 218 | 219 | ``` 220 | ansible-playbook -i hosts main.yml 221 | ``` 222 | 223 | Let's keep the app both versions running for 15 minutes, only to collect more data to our Grafana dashboard. 224 | 225 | ![Canary app version 2.0.0](images/dash-2.png) 226 | 227 | 228 | Ok, everything is right and working with our new version. Now, we want to replace entire the version 1.0.0 by the version 2.0.0. 229 | 230 | 231 | Let's to deploy the app version 2.0.0: 232 | 233 | ``` 234 | cd k8s-canary-deploy-example/deploy-app-v2-playbook/ 235 | ``` 236 | 237 | Execute the playbook: 238 | 239 | ``` 240 | ansible-playbook -i hosts main.yml 241 | ``` 242 | 243 | 244 | ![Deploy app version 2.0.0](images/dash-3.png) 245 | 246 | ![Deploy app version 2.0.0](images/dash-3.1.png) 247 | 248 | 249 | Great! Now we have the new version running in production and the best part, using canary deployment to prevent downtime of our app. 250 | 251 | 252 | ### TODO LIST 253 | - Create a PV and a PVC to Grafana and Prometheus 254 | - Create a pipeline using Jenkins/Gitlab 255 | - Create new dashboards 256 | -------------------------------------------------------------------------------- /nginx-prometheus-exporter-v1/lib/prometheus.lua: -------------------------------------------------------------------------------- 1 | -- vim: ts=2:sw=2:sts=2:expandtab 2 | -- 3 | -- This module uses a single dictionary shared between Nginx workers to keep 4 | -- all metrics. Each counter is stored as a separate entry in that dictionary, 5 | -- which allows us to increment them using built-in `incr` method. 6 | -- 7 | -- Prometheus requires that (a) all samples for a given metric are presented 8 | -- as one uninterrupted group, and (b) buckets of a histogram appear in 9 | -- increasing numerical order. We satisfy that by carefully constructing full 10 | -- metric names (i.e. metric name along with all labels) so that they meet 11 | -- those requirements while being sorted alphabetically. In particular: 12 | -- 13 | -- * all labels for a given metric are presented in reproducible order (the one 14 | -- used when labels were declared). "le" label for histogram metrics always 15 | -- goes last; 16 | -- * bucket boundaries (which are exposed as values of the "le" label) are 17 | -- presented as floating point numbers with leading and trailing zeroes. 18 | -- Number of of zeroes is determined for each bucketer automatically based on 19 | -- bucket boundaries; 20 | -- * internally "+Inf" bucket is stored as "Inf" (to make it appear after 21 | -- all numeric buckets), and gets replaced by "+Inf" just before we 22 | -- expose the metrics. 23 | -- 24 | -- For example, if you define your bucket boundaries as {0.00005, 10, 1000} 25 | -- then we will keep the following samples for a metric `m1` with label 26 | -- `site` set to `site1`: 27 | -- 28 | -- m1_bucket{site="site1",le="0000.00005"} 29 | -- m1_bucket{site="site1",le="0010.00000"} 30 | -- m1_bucket{site="site1",le="1000.00000"} 31 | -- m1_bucket{site="site1",le="Inf"} 32 | -- m1_count{site="site1"} 33 | -- m1_sum{site="site1"} 34 | -- 35 | -- "Inf" will be replaced by "+Inf" while publishing metrics. 36 | -- 37 | -- You can find the latest version and documentation at 38 | -- https://github.com/knyar/nginx-lua-prometheus 39 | -- Released under MIT license. 40 | 41 | 42 | -- Default set of latency buckets, 5ms to 10s: 43 | local DEFAULT_BUCKETS = {0.005, 0.01, 0.02, 0.03, 0.05, 0.075, 0.1, 0.2, 0.3, 44 | 0.4, 0.5, 0.75, 1, 1.5, 2, 3, 4, 5, 10} 45 | 46 | -- Metric is a "parent class" for all metrics. 47 | local Metric = {} 48 | function Metric:new(o) 49 | o = o or {} 50 | setmetatable(o, self) 51 | self.__index = self 52 | return o 53 | end 54 | 55 | -- Checks that the right number of labels values have been passed. 56 | -- 57 | -- Args: 58 | -- label_values: an array of label values. 59 | -- 60 | -- Returns: 61 | -- an error message or nil 62 | function Metric:check_labels(label_values) 63 | if self.label_names == nil and label_values == nil then 64 | return 65 | elseif self.label_names == nil and label_values ~= nil then 66 | return "Expected no labels for " .. self.name .. ", got " .. #label_values 67 | elseif label_values == nil and self.label_names ~= nil then 68 | return "Expected " .. #self.label_names .. " labels for " .. 69 | self.name .. ", got none" 70 | elseif #self.label_names ~= #label_values then 71 | return "Wrong number of labels for " .. self.name .. ". Expected " .. 72 | #self.label_names .. ", got " .. #label_values 73 | else 74 | for i, k in ipairs(self.label_names) do 75 | if label_values[i] == nil then 76 | return "Unexpected nil value for label " .. k .. " of " .. self.name 77 | end 78 | end 79 | end 80 | end 81 | 82 | local Counter = Metric:new() 83 | -- Increase a given counter by `value` 84 | -- 85 | -- Args: 86 | -- value: (number) a value to add to the counter. Defaults to 1 if skipped. 87 | -- label_values: an array of label values. Can be nil (i.e. not defined) for 88 | -- metrics that have no labels. 89 | function Counter:inc(value, label_values) 90 | local err = self:check_labels(label_values) 91 | if err ~= nil then 92 | self.prometheus:log_error(err) 93 | return 94 | end 95 | self.prometheus:inc(self.name, self.label_names, label_values, value or 1) 96 | end 97 | 98 | local Gauge = Metric:new() 99 | -- Set a given gauge to `value` 100 | -- 101 | -- Args: 102 | -- value: (number) a value to set the gauge to. Should be defined. 103 | -- label_values: an array of label values. Can be nil (i.e. not defined) for 104 | -- metrics that have no labels. 105 | function Gauge:set(value, label_values) 106 | if value == nil then 107 | self.prometheus:log_error("No value passed for " .. self.name) 108 | return 109 | end 110 | local err = self:check_labels(label_values) 111 | if err ~= nil then 112 | self.prometheus:log_error(err) 113 | return 114 | end 115 | self.prometheus:set(self.name, self.label_names, label_values, value) 116 | end 117 | 118 | local Histogram = Metric:new() 119 | -- Record a given value in a histogram. 120 | -- 121 | -- Args: 122 | -- value: (number) a value to record. Should be defined. 123 | -- label_values: an array of label values. Can be nil (i.e. not defined) for 124 | -- metrics that have no labels. 125 | function Histogram:observe(value, label_values) 126 | if value == nil then 127 | self.prometheus:log_error("No value passed for " .. self.name) 128 | return 129 | end 130 | local err = self:check_labels(label_values) 131 | if err ~= nil then 132 | self.prometheus:log_error(err) 133 | return 134 | end 135 | self.prometheus:histogram_observe(self.name, self.label_names, label_values, value) 136 | end 137 | 138 | local Prometheus = {} 139 | Prometheus.__index = Prometheus 140 | Prometheus.initialized = false 141 | 142 | -- Generate full metric name that includes all labels. 143 | -- 144 | -- Args: 145 | -- name: string 146 | -- label_names: (array) a list of label keys. 147 | -- label_values: (array) a list of label values. 148 | -- Returns: 149 | -- (string) full metric name. 150 | local function full_metric_name(name, label_names, label_values) 151 | if not label_names then 152 | return name 153 | end 154 | local label_parts = {} 155 | for idx, key in ipairs(label_names) do 156 | local label_value = (string.format("%s", label_values[idx]) 157 | :gsub("\\", "\\\\") 158 | :gsub("\n", "\\n") 159 | :gsub('"', '\\"')) 160 | table.insert(label_parts, key .. '="' .. label_value .. '"') 161 | end 162 | return name .. "{" .. table.concat(label_parts, ",") .. "}" 163 | end 164 | 165 | -- Construct bucket format for a list of buckets. 166 | -- 167 | -- This receives a list of buckets and returns a sprintf template that should 168 | -- be used for bucket boundaries to make them come in increasing order when 169 | -- sorted alphabetically. 170 | -- 171 | -- To re-phrase, this is where we detect how many leading and trailing zeros we 172 | -- need. 173 | -- 174 | -- Args: 175 | -- buckets: a list of buckets 176 | -- 177 | -- Returns: 178 | -- (string) a sprintf template. 179 | local function construct_bucket_format(buckets) 180 | local max_order = 1 181 | local max_precision = 1 182 | for _, bucket in ipairs(buckets) do 183 | assert(type(bucket) == "number", "bucket boundaries should be numeric") 184 | -- floating point number with all trailing zeros removed 185 | local as_string = string.format("%f", bucket):gsub("0*$", "") 186 | local dot_idx = as_string:find(".", 1, true) 187 | max_order = math.max(max_order, dot_idx - 1) 188 | max_precision = math.max(max_precision, as_string:len() - dot_idx) 189 | end 190 | return "%0" .. (max_order + max_precision + 1) .. "." .. max_precision .. "f" 191 | end 192 | 193 | -- Extract short metric name from the full one. 194 | -- 195 | -- Args: 196 | -- full_name: (string) full metric name that can include labels. 197 | -- 198 | -- Returns: 199 | -- (string) short metric name with no labels. For a `*_bucket` metric of 200 | -- histogram the _bucket suffix will be removed. 201 | local function short_metric_name(full_name) 202 | local labels_start, _ = full_name:find("{") 203 | if not labels_start then 204 | -- no labels 205 | return full_name 206 | end 207 | local suffix_idx, _ = full_name:find("_bucket{") 208 | if suffix_idx and full_name:find("le=") then 209 | -- this is a histogram metric 210 | return full_name:sub(1, suffix_idx - 1) 211 | end 212 | -- this is not a histogram metric 213 | return full_name:sub(1, labels_start - 1) 214 | end 215 | 216 | -- Makes a shallow copy of a table 217 | local function copy_table(table) 218 | local new = {} 219 | if table ~= nil then 220 | for k, v in ipairs(table) do 221 | new[k] = v 222 | end 223 | end 224 | return new 225 | end 226 | 227 | -- Initialize the module. 228 | -- 229 | -- This should be called once from the `init_by_lua` section in nginx 230 | -- configuration. 231 | -- 232 | -- Args: 233 | -- dict_name: (string) name of the nginx shared dictionary which will be 234 | -- used to store all metrics 235 | -- prefix: (optional string) if supplied, prefix is added to all 236 | -- metric names on output 237 | -- 238 | -- Returns: 239 | -- an object that should be used to register metrics. 240 | function Prometheus.init(dict_name, prefix) 241 | local self = setmetatable({}, Prometheus) 242 | self.dict = ngx.shared[dict_name or "prometheus_metrics"] 243 | self.help = {} 244 | if prefix then 245 | self.prefix = prefix 246 | else 247 | self.prefix = '' 248 | end 249 | self.type = {} 250 | self.registered = {} 251 | self.buckets = {} 252 | self.bucket_format = {} 253 | self.initialized = true 254 | 255 | self:counter("nginx_metric_errors_total", 256 | "Number of nginx-lua-prometheus errors") 257 | self.dict:set("nginx_metric_errors_total", 0) 258 | return self 259 | end 260 | 261 | function Prometheus:log_error(...) 262 | ngx.log(ngx.ERR, ...) 263 | self.dict:incr("nginx_metric_errors_total", 1) 264 | end 265 | 266 | function Prometheus:log_error_kv(key, value, err) 267 | self:log_error( 268 | "Error while setting '", key, "' to '", value, "': '", err, "'") 269 | end 270 | 271 | -- Register a counter. 272 | -- 273 | -- Args: 274 | -- name: (string) name of the metric. Required. 275 | -- description: (string) description of the metric. Will be used for the HELP 276 | -- comment on the metrics page. Optional. 277 | -- label_names: array of strings, defining a list of metrics. Optional. 278 | -- 279 | -- Returns: 280 | -- a Counter object. 281 | function Prometheus:counter(name, description, label_names) 282 | if not self.initialized then 283 | ngx.log(ngx.ERR, "Prometheus module has not been initialized") 284 | return 285 | end 286 | 287 | if self.registered[name] then 288 | self:log_error("Duplicate metric " .. name) 289 | return 290 | end 291 | self.registered[name] = true 292 | self.help[name] = description 293 | self.type[name] = "counter" 294 | 295 | return Counter:new{name=name, label_names=label_names, prometheus=self} 296 | end 297 | 298 | -- Register a gauge. 299 | -- 300 | -- Args: 301 | -- name: (string) name of the metric. Required. 302 | -- description: (string) description of the metric. Will be used for the HELP 303 | -- comment on the metrics page. Optional. 304 | -- label_names: array of strings, defining a list of metrics. Optional. 305 | -- 306 | -- Returns: 307 | -- a Gauge object. 308 | function Prometheus:gauge(name, description, label_names) 309 | if not self.initialized then 310 | ngx.log(ngx.ERR, "Prometheus module has not been initialized") 311 | return 312 | end 313 | 314 | if self.registered[name] then 315 | self:log_error("Duplicate metric " .. name) 316 | return 317 | end 318 | self.registered[name] = true 319 | self.help[name] = description 320 | self.type[name] = "gauge" 321 | 322 | return Gauge:new{name=name, label_names=label_names, prometheus=self} 323 | end 324 | 325 | 326 | -- Register a histogram. 327 | -- 328 | -- Args: 329 | -- name: (string) name of the metric. Required. 330 | -- description: (string) description of the metric. Will be used for the HELP 331 | -- comment on the metrics page. Optional. 332 | -- label_names: array of strings, defining a list of metrics. Optional. 333 | -- buckets: array if numbers, defining bucket boundaries. Optional. 334 | -- 335 | -- Returns: 336 | -- a Histogram object. 337 | function Prometheus:histogram(name, description, label_names, buckets) 338 | if not self.initialized then 339 | ngx.log(ngx.ERR, "Prometheus module has not been initialized") 340 | return 341 | end 342 | 343 | for _, label_name in ipairs(label_names or {}) do 344 | if label_name == "le" then 345 | self:log_error("Invalid label name 'le' in " .. name) 346 | return 347 | end 348 | end 349 | 350 | for _, suffix in ipairs({"", "_bucket", "_count", "_sum"}) do 351 | if self.registered[name .. suffix] then 352 | self:log_error("Duplicate metric " .. name .. suffix) 353 | return 354 | end 355 | self.registered[name .. suffix] = true 356 | end 357 | self.help[name] = description 358 | self.type[name] = "histogram" 359 | 360 | self.buckets[name] = buckets or DEFAULT_BUCKETS 361 | self.bucket_format[name] = construct_bucket_format(self.buckets[name]) 362 | 363 | return Histogram:new{name=name, label_names=label_names, prometheus=self} 364 | end 365 | 366 | -- Set a given dictionary key. 367 | -- This overwrites existing values, so it should only be used when initializing 368 | -- metrics or when explicitely overwriting the previous value of a metric. 369 | function Prometheus:set_key(key, value) 370 | local ok, err = self.dict:safe_set(key, value) 371 | if not ok then 372 | self:log_error_kv(key, value, err) 373 | end 374 | end 375 | 376 | -- Increment a given counter by `value`. 377 | -- 378 | -- Args: 379 | -- name: (string) short metric name without any labels. 380 | -- label_names: (array) a list of label keys. 381 | -- label_values: (array) a list of label values. 382 | -- value: (number) value to add. Optional, defaults to 1. 383 | function Prometheus:inc(name, label_names, label_values, value) 384 | local key = full_metric_name(name, label_names, label_values) 385 | if value == nil then value = 1 end 386 | if value < 0 then 387 | self:log_error_kv(key, value, "Value should not be negative") 388 | return 389 | end 390 | 391 | local newval, err = self.dict:incr(key, value) 392 | if newval then 393 | return 394 | end 395 | -- Yes, this looks like a race, so I guess we might under-report some values 396 | -- when multiple workers simultaneously try to create the same metric. 397 | -- Hopefully this does not happen too often (shared dictionary does not get 398 | -- reset during configuation reload). 399 | if err == "not found" then 400 | self:set_key(key, value) 401 | return 402 | end 403 | -- Unexpected error 404 | self:log_error_kv(key, value, err) 405 | end 406 | 407 | -- Set the current value of a gauge to `value` 408 | -- 409 | -- Args: 410 | -- name: (string) short metric name without any labels. 411 | -- label_names: (array) a list of label keys. 412 | -- label_values: (array) a list of label values. 413 | -- value: (number) the new value for the gauge. 414 | function Prometheus:set(name, label_names, label_values, value) 415 | local key = full_metric_name(name, label_names, label_values) 416 | self:set_key(key, value) 417 | end 418 | 419 | -- Record a given value into a histogram metric. 420 | -- 421 | -- Args: 422 | -- name: (string) short metric name without any labels. 423 | -- label_names: (array) a list of label keys. 424 | -- label_values: (array) a list of label values. 425 | -- value: (number) value to observe. 426 | function Prometheus:histogram_observe(name, label_names, label_values, value) 427 | self:inc(name .. "_count", label_names, label_values, 1) 428 | self:inc(name .. "_sum", label_names, label_values, value) 429 | 430 | -- we are going to mutate arrays of label names and values, so create a copy. 431 | local l_names = copy_table(label_names) 432 | local l_values = copy_table(label_values) 433 | 434 | -- Last bucket. Note, that the label value is "Inf" rather than "+Inf" 435 | -- required by Prometheus. This is necessary for this bucket to be the last 436 | -- one when all metrics are lexicographically sorted. "Inf" will get replaced 437 | -- by "+Inf" in Prometheus:collect(). 438 | table.insert(l_names, "le") 439 | table.insert(l_values, "Inf") 440 | self:inc(name .. "_bucket", l_names, l_values, 1) 441 | 442 | local label_count = #l_names 443 | for _, bucket in ipairs(self.buckets[name]) do 444 | if value <= bucket then 445 | -- last label is now "le" 446 | l_values[label_count] = self.bucket_format[name]:format(bucket) 447 | self:inc(name .. "_bucket", l_names, l_values, 1) 448 | end 449 | end 450 | end 451 | 452 | -- Present all metrics in a text format compatible with Prometheus. 453 | -- 454 | -- This function should be used to expose the metrics on a separate HTTP page. 455 | -- It will get the metrics from the dictionary, sort them, and expose them 456 | -- aling with TYPE and HELP comments. 457 | function Prometheus:collect() 458 | ngx.header.content_type = "text/plain" 459 | if not self.initialized then 460 | ngx.log(ngx.ERR, "Prometheus module has not been initialized") 461 | return 462 | end 463 | 464 | local keys = self.dict:get_keys(0) 465 | -- Prometheus server expects buckets of a histogram to appear in increasing 466 | -- numerical order of their label values. 467 | table.sort(keys) 468 | 469 | local seen_metrics = {} 470 | for _, key in ipairs(keys) do 471 | local value, err = self.dict:get(key) 472 | if value then 473 | local short_name = short_metric_name(key) 474 | if not seen_metrics[short_name] then 475 | if self.help[short_name] then 476 | ngx.say("# HELP " .. self.prefix .. short_name .. " " .. self.help[short_name]) 477 | end 478 | if self.type[short_name] then 479 | ngx.say("# TYPE " .. self.prefix .. short_name .. " " .. self.type[short_name]) 480 | end 481 | seen_metrics[short_name] = true 482 | end 483 | -- Replace "Inf" with "+Inf" in each metric's last bucket 'le' label. 484 | ngx.say(self.prefix .. key:gsub('le="Inf"', 'le="+Inf"'), " ", value) 485 | else 486 | self:log_error("Error getting '", key, "': ", err) 487 | end 488 | end 489 | end 490 | 491 | return Prometheus 492 | -------------------------------------------------------------------------------- /nginx-prometheus-exporter-v2/lib/prometheus.lua: -------------------------------------------------------------------------------- 1 | -- vim: ts=2:sw=2:sts=2:expandtab 2 | -- 3 | -- This module uses a single dictionary shared between Nginx workers to keep 4 | -- all metrics. Each counter is stored as a separate entry in that dictionary, 5 | -- which allows us to increment them using built-in `incr` method. 6 | -- 7 | -- Prometheus requires that (a) all samples for a given metric are presented 8 | -- as one uninterrupted group, and (b) buckets of a histogram appear in 9 | -- increasing numerical order. We satisfy that by carefully constructing full 10 | -- metric names (i.e. metric name along with all labels) so that they meet 11 | -- those requirements while being sorted alphabetically. In particular: 12 | -- 13 | -- * all labels for a given metric are presented in reproducible order (the one 14 | -- used when labels were declared). "le" label for histogram metrics always 15 | -- goes last; 16 | -- * bucket boundaries (which are exposed as values of the "le" label) are 17 | -- presented as floating point numbers with leading and trailing zeroes. 18 | -- Number of of zeroes is determined for each bucketer automatically based on 19 | -- bucket boundaries; 20 | -- * internally "+Inf" bucket is stored as "Inf" (to make it appear after 21 | -- all numeric buckets), and gets replaced by "+Inf" just before we 22 | -- expose the metrics. 23 | -- 24 | -- For example, if you define your bucket boundaries as {0.00005, 10, 1000} 25 | -- then we will keep the following samples for a metric `m1` with label 26 | -- `site` set to `site1`: 27 | -- 28 | -- m1_bucket{site="site1",le="0000.00005"} 29 | -- m1_bucket{site="site1",le="0010.00000"} 30 | -- m1_bucket{site="site1",le="1000.00000"} 31 | -- m1_bucket{site="site1",le="Inf"} 32 | -- m1_count{site="site1"} 33 | -- m1_sum{site="site1"} 34 | -- 35 | -- "Inf" will be replaced by "+Inf" while publishing metrics. 36 | -- 37 | -- You can find the latest version and documentation at 38 | -- https://github.com/knyar/nginx-lua-prometheus 39 | -- Released under MIT license. 40 | 41 | 42 | -- Default set of latency buckets, 5ms to 10s: 43 | local DEFAULT_BUCKETS = {0.005, 0.01, 0.02, 0.03, 0.05, 0.075, 0.1, 0.2, 0.3, 44 | 0.4, 0.5, 0.75, 1, 1.5, 2, 3, 4, 5, 10} 45 | 46 | -- Metric is a "parent class" for all metrics. 47 | local Metric = {} 48 | function Metric:new(o) 49 | o = o or {} 50 | setmetatable(o, self) 51 | self.__index = self 52 | return o 53 | end 54 | 55 | -- Checks that the right number of labels values have been passed. 56 | -- 57 | -- Args: 58 | -- label_values: an array of label values. 59 | -- 60 | -- Returns: 61 | -- an error message or nil 62 | function Metric:check_labels(label_values) 63 | if self.label_names == nil and label_values == nil then 64 | return 65 | elseif self.label_names == nil and label_values ~= nil then 66 | return "Expected no labels for " .. self.name .. ", got " .. #label_values 67 | elseif label_values == nil and self.label_names ~= nil then 68 | return "Expected " .. #self.label_names .. " labels for " .. 69 | self.name .. ", got none" 70 | elseif #self.label_names ~= #label_values then 71 | return "Wrong number of labels for " .. self.name .. ". Expected " .. 72 | #self.label_names .. ", got " .. #label_values 73 | else 74 | for i, k in ipairs(self.label_names) do 75 | if label_values[i] == nil then 76 | return "Unexpected nil value for label " .. k .. " of " .. self.name 77 | end 78 | end 79 | end 80 | end 81 | 82 | local Counter = Metric:new() 83 | -- Increase a given counter by `value` 84 | -- 85 | -- Args: 86 | -- value: (number) a value to add to the counter. Defaults to 1 if skipped. 87 | -- label_values: an array of label values. Can be nil (i.e. not defined) for 88 | -- metrics that have no labels. 89 | function Counter:inc(value, label_values) 90 | local err = self:check_labels(label_values) 91 | if err ~= nil then 92 | self.prometheus:log_error(err) 93 | return 94 | end 95 | self.prometheus:inc(self.name, self.label_names, label_values, value or 1) 96 | end 97 | 98 | local Gauge = Metric:new() 99 | -- Set a given gauge to `value` 100 | -- 101 | -- Args: 102 | -- value: (number) a value to set the gauge to. Should be defined. 103 | -- label_values: an array of label values. Can be nil (i.e. not defined) for 104 | -- metrics that have no labels. 105 | function Gauge:set(value, label_values) 106 | if value == nil then 107 | self.prometheus:log_error("No value passed for " .. self.name) 108 | return 109 | end 110 | local err = self:check_labels(label_values) 111 | if err ~= nil then 112 | self.prometheus:log_error(err) 113 | return 114 | end 115 | self.prometheus:set(self.name, self.label_names, label_values, value) 116 | end 117 | 118 | local Histogram = Metric:new() 119 | -- Record a given value in a histogram. 120 | -- 121 | -- Args: 122 | -- value: (number) a value to record. Should be defined. 123 | -- label_values: an array of label values. Can be nil (i.e. not defined) for 124 | -- metrics that have no labels. 125 | function Histogram:observe(value, label_values) 126 | if value == nil then 127 | self.prometheus:log_error("No value passed for " .. self.name) 128 | return 129 | end 130 | local err = self:check_labels(label_values) 131 | if err ~= nil then 132 | self.prometheus:log_error(err) 133 | return 134 | end 135 | self.prometheus:histogram_observe(self.name, self.label_names, label_values, value) 136 | end 137 | 138 | local Prometheus = {} 139 | Prometheus.__index = Prometheus 140 | Prometheus.initialized = false 141 | 142 | -- Generate full metric name that includes all labels. 143 | -- 144 | -- Args: 145 | -- name: string 146 | -- label_names: (array) a list of label keys. 147 | -- label_values: (array) a list of label values. 148 | -- Returns: 149 | -- (string) full metric name. 150 | local function full_metric_name(name, label_names, label_values) 151 | if not label_names then 152 | return name 153 | end 154 | local label_parts = {} 155 | for idx, key in ipairs(label_names) do 156 | local label_value = (string.format("%s", label_values[idx]) 157 | :gsub("\\", "\\\\") 158 | :gsub("\n", "\\n") 159 | :gsub('"', '\\"')) 160 | table.insert(label_parts, key .. '="' .. label_value .. '"') 161 | end 162 | return name .. "{" .. table.concat(label_parts, ",") .. "}" 163 | end 164 | 165 | -- Construct bucket format for a list of buckets. 166 | -- 167 | -- This receives a list of buckets and returns a sprintf template that should 168 | -- be used for bucket boundaries to make them come in increasing order when 169 | -- sorted alphabetically. 170 | -- 171 | -- To re-phrase, this is where we detect how many leading and trailing zeros we 172 | -- need. 173 | -- 174 | -- Args: 175 | -- buckets: a list of buckets 176 | -- 177 | -- Returns: 178 | -- (string) a sprintf template. 179 | local function construct_bucket_format(buckets) 180 | local max_order = 1 181 | local max_precision = 1 182 | for _, bucket in ipairs(buckets) do 183 | assert(type(bucket) == "number", "bucket boundaries should be numeric") 184 | -- floating point number with all trailing zeros removed 185 | local as_string = string.format("%f", bucket):gsub("0*$", "") 186 | local dot_idx = as_string:find(".", 1, true) 187 | max_order = math.max(max_order, dot_idx - 1) 188 | max_precision = math.max(max_precision, as_string:len() - dot_idx) 189 | end 190 | return "%0" .. (max_order + max_precision + 1) .. "." .. max_precision .. "f" 191 | end 192 | 193 | -- Extract short metric name from the full one. 194 | -- 195 | -- Args: 196 | -- full_name: (string) full metric name that can include labels. 197 | -- 198 | -- Returns: 199 | -- (string) short metric name with no labels. For a `*_bucket` metric of 200 | -- histogram the _bucket suffix will be removed. 201 | local function short_metric_name(full_name) 202 | local labels_start, _ = full_name:find("{") 203 | if not labels_start then 204 | -- no labels 205 | return full_name 206 | end 207 | local suffix_idx, _ = full_name:find("_bucket{") 208 | if suffix_idx and full_name:find("le=") then 209 | -- this is a histogram metric 210 | return full_name:sub(1, suffix_idx - 1) 211 | end 212 | -- this is not a histogram metric 213 | return full_name:sub(1, labels_start - 1) 214 | end 215 | 216 | -- Makes a shallow copy of a table 217 | local function copy_table(table) 218 | local new = {} 219 | if table ~= nil then 220 | for k, v in ipairs(table) do 221 | new[k] = v 222 | end 223 | end 224 | return new 225 | end 226 | 227 | -- Initialize the module. 228 | -- 229 | -- This should be called once from the `init_by_lua` section in nginx 230 | -- configuration. 231 | -- 232 | -- Args: 233 | -- dict_name: (string) name of the nginx shared dictionary which will be 234 | -- used to store all metrics 235 | -- prefix: (optional string) if supplied, prefix is added to all 236 | -- metric names on output 237 | -- 238 | -- Returns: 239 | -- an object that should be used to register metrics. 240 | function Prometheus.init(dict_name, prefix) 241 | local self = setmetatable({}, Prometheus) 242 | self.dict = ngx.shared[dict_name or "prometheus_metrics"] 243 | self.help = {} 244 | if prefix then 245 | self.prefix = prefix 246 | else 247 | self.prefix = '' 248 | end 249 | self.type = {} 250 | self.registered = {} 251 | self.buckets = {} 252 | self.bucket_format = {} 253 | self.initialized = true 254 | 255 | self:counter("nginx_metric_errors_total", 256 | "Number of nginx-lua-prometheus errors") 257 | self.dict:set("nginx_metric_errors_total", 0) 258 | return self 259 | end 260 | 261 | function Prometheus:log_error(...) 262 | ngx.log(ngx.ERR, ...) 263 | self.dict:incr("nginx_metric_errors_total", 1) 264 | end 265 | 266 | function Prometheus:log_error_kv(key, value, err) 267 | self:log_error( 268 | "Error while setting '", key, "' to '", value, "': '", err, "'") 269 | end 270 | 271 | -- Register a counter. 272 | -- 273 | -- Args: 274 | -- name: (string) name of the metric. Required. 275 | -- description: (string) description of the metric. Will be used for the HELP 276 | -- comment on the metrics page. Optional. 277 | -- label_names: array of strings, defining a list of metrics. Optional. 278 | -- 279 | -- Returns: 280 | -- a Counter object. 281 | function Prometheus:counter(name, description, label_names) 282 | if not self.initialized then 283 | ngx.log(ngx.ERR, "Prometheus module has not been initialized") 284 | return 285 | end 286 | 287 | if self.registered[name] then 288 | self:log_error("Duplicate metric " .. name) 289 | return 290 | end 291 | self.registered[name] = true 292 | self.help[name] = description 293 | self.type[name] = "counter" 294 | 295 | return Counter:new{name=name, label_names=label_names, prometheus=self} 296 | end 297 | 298 | -- Register a gauge. 299 | -- 300 | -- Args: 301 | -- name: (string) name of the metric. Required. 302 | -- description: (string) description of the metric. Will be used for the HELP 303 | -- comment on the metrics page. Optional. 304 | -- label_names: array of strings, defining a list of metrics. Optional. 305 | -- 306 | -- Returns: 307 | -- a Gauge object. 308 | function Prometheus:gauge(name, description, label_names) 309 | if not self.initialized then 310 | ngx.log(ngx.ERR, "Prometheus module has not been initialized") 311 | return 312 | end 313 | 314 | if self.registered[name] then 315 | self:log_error("Duplicate metric " .. name) 316 | return 317 | end 318 | self.registered[name] = true 319 | self.help[name] = description 320 | self.type[name] = "gauge" 321 | 322 | return Gauge:new{name=name, label_names=label_names, prometheus=self} 323 | end 324 | 325 | 326 | -- Register a histogram. 327 | -- 328 | -- Args: 329 | -- name: (string) name of the metric. Required. 330 | -- description: (string) description of the metric. Will be used for the HELP 331 | -- comment on the metrics page. Optional. 332 | -- label_names: array of strings, defining a list of metrics. Optional. 333 | -- buckets: array if numbers, defining bucket boundaries. Optional. 334 | -- 335 | -- Returns: 336 | -- a Histogram object. 337 | function Prometheus:histogram(name, description, label_names, buckets) 338 | if not self.initialized then 339 | ngx.log(ngx.ERR, "Prometheus module has not been initialized") 340 | return 341 | end 342 | 343 | for _, label_name in ipairs(label_names or {}) do 344 | if label_name == "le" then 345 | self:log_error("Invalid label name 'le' in " .. name) 346 | return 347 | end 348 | end 349 | 350 | for _, suffix in ipairs({"", "_bucket", "_count", "_sum"}) do 351 | if self.registered[name .. suffix] then 352 | self:log_error("Duplicate metric " .. name .. suffix) 353 | return 354 | end 355 | self.registered[name .. suffix] = true 356 | end 357 | self.help[name] = description 358 | self.type[name] = "histogram" 359 | 360 | self.buckets[name] = buckets or DEFAULT_BUCKETS 361 | self.bucket_format[name] = construct_bucket_format(self.buckets[name]) 362 | 363 | return Histogram:new{name=name, label_names=label_names, prometheus=self} 364 | end 365 | 366 | -- Set a given dictionary key. 367 | -- This overwrites existing values, so it should only be used when initializing 368 | -- metrics or when explicitely overwriting the previous value of a metric. 369 | function Prometheus:set_key(key, value) 370 | local ok, err = self.dict:safe_set(key, value) 371 | if not ok then 372 | self:log_error_kv(key, value, err) 373 | end 374 | end 375 | 376 | -- Increment a given counter by `value`. 377 | -- 378 | -- Args: 379 | -- name: (string) short metric name without any labels. 380 | -- label_names: (array) a list of label keys. 381 | -- label_values: (array) a list of label values. 382 | -- value: (number) value to add. Optional, defaults to 1. 383 | function Prometheus:inc(name, label_names, label_values, value) 384 | local key = full_metric_name(name, label_names, label_values) 385 | if value == nil then value = 1 end 386 | if value < 0 then 387 | self:log_error_kv(key, value, "Value should not be negative") 388 | return 389 | end 390 | 391 | local newval, err = self.dict:incr(key, value) 392 | if newval then 393 | return 394 | end 395 | -- Yes, this looks like a race, so I guess we might under-report some values 396 | -- when multiple workers simultaneously try to create the same metric. 397 | -- Hopefully this does not happen too often (shared dictionary does not get 398 | -- reset during configuation reload). 399 | if err == "not found" then 400 | self:set_key(key, value) 401 | return 402 | end 403 | -- Unexpected error 404 | self:log_error_kv(key, value, err) 405 | end 406 | 407 | -- Set the current value of a gauge to `value` 408 | -- 409 | -- Args: 410 | -- name: (string) short metric name without any labels. 411 | -- label_names: (array) a list of label keys. 412 | -- label_values: (array) a list of label values. 413 | -- value: (number) the new value for the gauge. 414 | function Prometheus:set(name, label_names, label_values, value) 415 | local key = full_metric_name(name, label_names, label_values) 416 | self:set_key(key, value) 417 | end 418 | 419 | -- Record a given value into a histogram metric. 420 | -- 421 | -- Args: 422 | -- name: (string) short metric name without any labels. 423 | -- label_names: (array) a list of label keys. 424 | -- label_values: (array) a list of label values. 425 | -- value: (number) value to observe. 426 | function Prometheus:histogram_observe(name, label_names, label_values, value) 427 | self:inc(name .. "_count", label_names, label_values, 1) 428 | self:inc(name .. "_sum", label_names, label_values, value) 429 | 430 | -- we are going to mutate arrays of label names and values, so create a copy. 431 | local l_names = copy_table(label_names) 432 | local l_values = copy_table(label_values) 433 | 434 | -- Last bucket. Note, that the label value is "Inf" rather than "+Inf" 435 | -- required by Prometheus. This is necessary for this bucket to be the last 436 | -- one when all metrics are lexicographically sorted. "Inf" will get replaced 437 | -- by "+Inf" in Prometheus:collect(). 438 | table.insert(l_names, "le") 439 | table.insert(l_values, "Inf") 440 | self:inc(name .. "_bucket", l_names, l_values, 1) 441 | 442 | local label_count = #l_names 443 | for _, bucket in ipairs(self.buckets[name]) do 444 | if value <= bucket then 445 | -- last label is now "le" 446 | l_values[label_count] = self.bucket_format[name]:format(bucket) 447 | self:inc(name .. "_bucket", l_names, l_values, 1) 448 | end 449 | end 450 | end 451 | 452 | -- Present all metrics in a text format compatible with Prometheus. 453 | -- 454 | -- This function should be used to expose the metrics on a separate HTTP page. 455 | -- It will get the metrics from the dictionary, sort them, and expose them 456 | -- aling with TYPE and HELP comments. 457 | function Prometheus:collect() 458 | ngx.header.content_type = "text/plain" 459 | if not self.initialized then 460 | ngx.log(ngx.ERR, "Prometheus module has not been initialized") 461 | return 462 | end 463 | 464 | local keys = self.dict:get_keys(0) 465 | -- Prometheus server expects buckets of a histogram to appear in increasing 466 | -- numerical order of their label values. 467 | table.sort(keys) 468 | 469 | local seen_metrics = {} 470 | for _, key in ipairs(keys) do 471 | local value, err = self.dict:get(key) 472 | if value then 473 | local short_name = short_metric_name(key) 474 | if not seen_metrics[short_name] then 475 | if self.help[short_name] then 476 | ngx.say("# HELP " .. self.prefix .. short_name .. " " .. self.help[short_name]) 477 | end 478 | if self.type[short_name] then 479 | ngx.say("# TYPE " .. self.prefix .. short_name .. " " .. self.type[short_name]) 480 | end 481 | seen_metrics[short_name] = true 482 | end 483 | -- Replace "Inf" with "+Inf" in each metric's last bucket 'le' label. 484 | ngx.say(self.prefix .. key:gsub('le="Inf"', 'le="+Inf"'), " ", value) 485 | else 486 | self:log_error("Error getting '", key, "': ", err) 487 | end 488 | end 489 | end 490 | 491 | return Prometheus 492 | --------------------------------------------------------------------------------