├── sketches ├── tls.png ├── ui.png └── istio.png ├── helm ├── templates │ ├── serviceaccount.yaml │ ├── role.yaml │ ├── servicemonitor.yaml │ ├── rolebinding.yaml │ ├── ingress.yaml │ ├── istio_gateway.yaml │ ├── secret.yaml │ ├── istio_mtls.yaml │ ├── service.yaml │ ├── istio_serviceentry.yaml │ ├── service_internal.yaml │ ├── service-discovery.yaml │ ├── istio_destrules.yaml │ ├── configmap.yaml │ ├── _helpers.tpl │ ├── NOTES.txt │ ├── alerts.yaml │ └── statefulset.yaml ├── Chart.yaml └── values.yaml ├── LICENSE └── README.md /sketches/tls.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/arielb135/RabbitMQ-with-istio-MTLS/HEAD/sketches/tls.png -------------------------------------------------------------------------------- /sketches/ui.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/arielb135/RabbitMQ-with-istio-MTLS/HEAD/sketches/ui.png -------------------------------------------------------------------------------- /sketches/istio.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/arielb135/RabbitMQ-with-istio-MTLS/HEAD/sketches/istio.png -------------------------------------------------------------------------------- /helm/templates/serviceaccount.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.serviceAccount.create }} 2 | apiVersion: v1 3 | kind: ServiceAccount 4 | metadata: 5 | labels: 6 | app: {{ template "rabbitmq-ha.name" . }} 7 | chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" 8 | release: {{ .Release.Name | quote }} 9 | heritage: {{ .Release.Service | quote }} 10 | {{- if .Values.extraLabels }} 11 | {{ toYaml .Values.extraLabels | indent 4 }} 12 | {{- end }} 13 | name: {{ template "rabbitmq-ha.serviceAccountName" . }} 14 | {{- end }} 15 | -------------------------------------------------------------------------------- /helm/templates/role.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.rbac.create }} 2 | apiVersion: rbac.authorization.k8s.io/v1beta1 3 | kind: Role 4 | metadata: 5 | labels: 6 | app: {{ template "rabbitmq-ha.name" . }} 7 | chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" 8 | release: {{ .Release.Name | quote }} 9 | heritage: {{ .Release.Service | quote }} 10 | {{- if .Values.extraLabels }} 11 | {{ toYaml .Values.extraLabels | indent 4 }} 12 | {{- end }} 13 | name: {{ template "rabbitmq-ha.fullname" . }} 14 | rules: 15 | - apiGroups: [""] 16 | resources: ["endpoints"] 17 | verbs: ["get"] 18 | {{- end }} 19 | -------------------------------------------------------------------------------- /helm/Chart.yaml: -------------------------------------------------------------------------------- 1 | name: rabbitmq-ha 2 | apiVersion: v1 3 | appVersion: 3.7.4 4 | version: 1.12.1 5 | description: Highly available RabbitMQ cluster, the open source message broker 6 | software that implements the Advanced Message Queuing Protocol (AMQP). 7 | keywords: 8 | - rabbitmq 9 | - message queue 10 | - AMQP 11 | - AMQPS 12 | - MQTT 13 | - STOMP 14 | home: https://www.rabbitmq.com 15 | icon: https://bitnami.com/assets/stacks/rabbitmq/img/rabbitmq-stack-220x234.png 16 | sources: 17 | - https://github.com/rabbitmq/rabbitmq 18 | - https://github.com/docker-library/rabbitmq 19 | maintainers: 20 | - name: etiennetremel 21 | email: etienne.tremel@container-solutions.com 22 | -------------------------------------------------------------------------------- /helm/templates/servicemonitor.yaml: -------------------------------------------------------------------------------- 1 | {{ if and .Values.prometheus.exporter.enabled .Values.prometheus.operator.enabled }} 2 | apiVersion: monitoring.coreos.com/v1 3 | kind: ServiceMonitor 4 | metadata: 5 | name: {{ template "rabbitmq-ha.fullname" . }} 6 | labels: 7 | {{- if .Values.prometheus.operator.serviceMonitor.selector }} 8 | {{ toYaml .Values.prometheus.operator.serviceMonitor.selector | indent 4 }} 9 | {{- end }} 10 | spec: 11 | selector: 12 | matchLabels: 13 | app: {{ template "rabbitmq-ha.name" . }} 14 | release: {{ .Release.Name }} 15 | endpoints: 16 | - port: exporter 17 | interval: {{ .Values.prometheus.operator.serviceMonitor.interval }} 18 | namespaceSelector: 19 | any: true 20 | {{ end }} 21 | -------------------------------------------------------------------------------- /helm/templates/rolebinding.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.rbac.create }} 2 | apiVersion: rbac.authorization.k8s.io/v1beta1 3 | kind: RoleBinding 4 | metadata: 5 | labels: 6 | app: {{ template "rabbitmq-ha.name" . }} 7 | chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" 8 | release: {{ .Release.Name | quote }} 9 | heritage: {{ .Release.Service | quote }} 10 | {{- if .Values.extraLabels }} 11 | {{ toYaml .Values.extraLabels | indent 4 }} 12 | {{- end }} 13 | name: {{ template "rabbitmq-ha.fullname" . }} 14 | subjects: 15 | - kind: ServiceAccount 16 | name: {{ template "rabbitmq-ha.serviceAccountName" . }} 17 | roleRef: 18 | apiGroup: rbac.authorization.k8s.io 19 | kind: Role 20 | name: {{ template "rabbitmq-ha.fullname" . }} 21 | {{- end }} 22 | -------------------------------------------------------------------------------- /helm/templates/ingress.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.ingress.enabled }} 2 | apiVersion: extensions/v1beta1 3 | kind: Ingress 4 | metadata: 5 | name: {{ template "rabbitmq-ha.fullname" . }} 6 | labels: 7 | app: {{ template "rabbitmq-ha.name" . }} 8 | chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} 9 | release: {{ .Release.Name }} 10 | heritage: {{ .Release.Service }} 11 | annotations: 12 | {{- range $key, $value := .Values.ingress.annotations }} 13 | {{ $key }}: {{ $value | quote }} 14 | {{- end }} 15 | spec: 16 | rules: 17 | {{- if .Values.ingress.hostName }} 18 | - host: {{ .Values.ingress.hostName }} 19 | http: 20 | {{- else }} 21 | - http: 22 | {{- end }} 23 | paths: 24 | - path: {{ .Values.ingress.path }} 25 | backend: 26 | serviceName: {{ template "rabbitmq-ha.fullname" . }}-discovery 27 | servicePort: {{ .Values.rabbitmqManagerPort }} 28 | {{- if .Values.ingress.tls }} 29 | tls: 30 | - secretName: {{ .Values.ingress.tlsSecret }} 31 | {{- if .Values.ingress.hostName }} 32 | hosts: 33 | - {{ .Values.ingress.hostName }} 34 | {{- end }} 35 | {{- end }} 36 | {{- end }} 37 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | This is free and unencumbered software released into the public domain. 2 | 3 | Anyone is free to copy, modify, publish, use, compile, sell, or 4 | distribute this software, either in source code form or as a compiled 5 | binary, for any purpose, commercial or non-commercial, and by any 6 | means. 7 | 8 | In jurisdictions that recognize copyright laws, the author or authors 9 | of this software dedicate any and all copyright interest in the 10 | software to the public domain. We make this dedication for the benefit 11 | of the public at large and to the detriment of our heirs and 12 | successors. We intend this dedication to be an overt act of 13 | relinquishment in perpetuity of all present and future rights to this 14 | software under copyright law. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 19 | IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR 20 | OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 21 | ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 | OTHER DEALINGS IN THE SOFTWARE. 23 | 24 | For more information, please refer to 25 | -------------------------------------------------------------------------------- /helm/templates/istio_gateway.yaml: -------------------------------------------------------------------------------- 1 | {{- if and .Values.istio.enabled .Values.istio.ingress.enabled }} 2 | apiVersion: networking.istio.io/v1alpha3 3 | kind: Gateway 4 | metadata: 5 | name: {{ template "rabbitmq-ha.fullname" . }}-gw 6 | labels: 7 | app: {{ template "rabbitmq-ha.name" . }} 8 | chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} 9 | release: {{ .Release.Name }} 10 | heritage: {{ .Release.Service }} 11 | {{- if .Values.extraLabels }} 12 | {{ toYaml .Values.extraLabels | indent 4 }} 13 | {{- end }} 14 | spec: 15 | selector: 16 | istio: ingressgateway # use istio default controller 17 | servers: 18 | - port: 19 | number: 80 20 | name: http 21 | protocol: HTTP 22 | hosts: 23 | - {{ .Values.istio.ingress.managementHostName }} 24 | --- 25 | apiVersion: networking.istio.io/v1alpha3 26 | kind: VirtualService 27 | metadata: 28 | name: {{ template "rabbitmq-ha.fullname" . }} 29 | labels: 30 | app: {{ template "rabbitmq-ha.name" . }} 31 | chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} 32 | release: {{ .Release.Name }} 33 | heritage: {{ .Release.Service }} 34 | {{- if .Values.extraLabels }} 35 | {{ toYaml .Values.extraLabels | indent 4 }} 36 | {{- end }} 37 | spec: 38 | hosts: 39 | - {{ .Values.istio.ingress.managementHostName | quote }} 40 | gateways: 41 | - {{ template "rabbitmq-ha.fullname" . }}-gw 42 | http: 43 | - route: 44 | - destination: 45 | host: {{ template "rabbitmq-ha.fullname" . }}-internal.{{ .Release.Namespace }}.svc.cluster.local 46 | port: 47 | number: {{ .Values.rabbitmqManagerPort }} 48 | {{- end }} -------------------------------------------------------------------------------- /helm/templates/secret.yaml: -------------------------------------------------------------------------------- 1 | {{ if not .Values.existingSecret }} 2 | apiVersion: v1 3 | kind: Secret 4 | metadata: 5 | name: {{ template "rabbitmq-ha.fullname" . }} 6 | labels: 7 | app: {{ template "rabbitmq-ha.name" . }} 8 | chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" 9 | release: "{{ .Release.Name }}" 10 | heritage: "{{ .Release.Service }}" 11 | {{- if .Values.extraLabels }} 12 | {{ toYaml .Values.extraLabels | indent 4 }} 13 | {{- end }} 14 | type: Opaque 15 | data: 16 | {{- $password := .Values.rabbitmqPassword | default (randAlphaNum 24 | nospace) -}} 17 | {{- $_ := set .Values "rabbitmqPassword" $password }} 18 | rabbitmq-username: {{ .Values.rabbitmqUsername | b64enc | quote }} 19 | rabbitmq-password: {{ .Values.rabbitmqPassword | b64enc | quote }} 20 | rabbitmq-erlang-cookie: {{ .Values.rabbitmqErlangCookie | default (randAlphaNum 32) | b64enc | quote }} 21 | definitions.json: {{ include "rabbitmq-ha.definitions" . | b64enc | quote }} 22 | {{ end }} 23 | {{- if and .Values.rabbitmqCert.enabled (not .Values.rabbitmqCert.existingSecret) }} 24 | --- 25 | apiVersion: v1 26 | kind: Secret 27 | metadata: 28 | name: {{ template "rabbitmq-ha.fullname" . }}-cert 29 | labels: 30 | app: {{ template "rabbitmq-ha.name" . }} 31 | chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" 32 | release: "{{ .Release.Name }}" 33 | heritage: "{{ .Release.Service }}" 34 | {{- if .Values.extraLabels }} 35 | {{ toYaml .Values.extraLabels | indent 4 }} 36 | {{- end }} 37 | type: Opaque 38 | data: 39 | cacert.pem: {{ .Values.rabbitmqCert.cacertfile | quote }} 40 | cert.pem: {{ .Values.rabbitmqCert.certfile | quote }} 41 | key.pem: {{ .Values.rabbitmqCert.keyfile | quote }} 42 | {{- end }} 43 | -------------------------------------------------------------------------------- /helm/templates/istio_mtls.yaml: -------------------------------------------------------------------------------- 1 | {{- if and .Values.istio.enabled .Values.istio.mtls }} 2 | apiVersion: "authentication.istio.io/v1alpha1" 3 | kind: "Policy" 4 | metadata: 5 | name: "default" 6 | labels: 7 | app: {{ template "rabbitmq-ha.name" . }} 8 | chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} 9 | release: {{ .Release.Name }} 10 | heritage: {{ .Release.Service }} 11 | {{- if .Values.extraLabels }} 12 | {{ toYaml .Values.extraLabels | indent 4 }} 13 | {{- end }} 14 | spec: 15 | peers: 16 | - mtls: {} 17 | --- 18 | apiVersion: "authentication.istio.io/v1alpha1" 19 | kind: "Policy" 20 | metadata: 21 | name: {{ template "rabbitmq-ha.fullname" . }}-disable-mtls 22 | labels: 23 | app: {{ template "rabbitmq-ha.name" . }} 24 | chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} 25 | release: {{ .Release.Name }} 26 | heritage: {{ .Release.Service }} 27 | {{- if .Values.extraLabels }} 28 | {{ toYaml .Values.extraLabels | indent 4 }} 29 | {{- end }} 30 | spec: 31 | targets: 32 | - name: {{ template "rabbitmq-ha.fullname" . }}-discovery 33 | ports: 34 | - number: {{ .Values.Epmd.rabbitmqEpmdPort }} 35 | {{- if .Values.rabbitmqAmqpsSupport.enabled }} 36 | - number: {{ .Values.rabbitmqAmqpsSupport.amqpsNodePort }} 37 | {{- end }} 38 | {{- if .Values.rabbitmqAmqpsSupport.enabled }} 39 | - name: {{ template "rabbitmq-ha.fullname" . }} 40 | ports: 41 | - number: {{ .Values.rabbitmqAmqpsSupport.amqpsNodePort }} 42 | {{- end }} 43 | {{- if .Values.rabbitmqAmqpsSupport.enabled }} 44 | - name: {{ template "rabbitmq-ha.fullname" . }}-internal 45 | ports: 46 | - number: {{ .Values.rabbitmqAmqpsSupport.amqpsNodePort }} 47 | {{- end }} 48 | 49 | {{- end }} -------------------------------------------------------------------------------- /helm/templates/service.yaml: -------------------------------------------------------------------------------- 1 | {{- if or .Values.rabbitmqAmqpsSupport.enabled (and .Values.prometheus.exporter.enabled .Values.prometheus.exporter.exportOperator) }} 2 | apiVersion: v1 3 | kind: Service 4 | metadata: 5 | {{- if .Values.service.annotations_lb }} 6 | annotations: 7 | {{ toYaml .Values.service.annotations_lb | indent 4 }} 8 | {{- end }} 9 | name: {{ template "rabbitmq-ha.fullname" . }} 10 | labels: 11 | app: {{ template "rabbitmq-ha.name" . }} 12 | chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} 13 | release: {{ .Release.Name }} 14 | heritage: {{ .Release.Service }} 15 | {{- if .Values.extraLabels }} 16 | {{ toYaml .Values.extraLabels | indent 4 }} 17 | {{- end }} 18 | spec: 19 | clusterIP: "{{ .Values.service.clusterIP }}" 20 | {{- if .Values.service.externalIPs }} 21 | externalIPs: 22 | {{ toYaml .Values.service.externalIPs | indent 4 }} 23 | {{- end }} 24 | {{- if .Values.service.loadBalancerIP }} 25 | loadBalancerIP: "{{ .Values.service.loadBalancerIP }}" 26 | {{- end }} 27 | {{- if .Values.service.externalTrafficPolicy }} 28 | externalTrafficPolicy: "{{ .Values.service.externalTrafficPolicy }}" 29 | {{- end }} 30 | {{- if .Values.service.loadBalancerSourceRanges }} 31 | loadBalancerSourceRanges: 32 | {{ toYaml .Values.service.loadBalancerSourceRanges | indent 4 }} 33 | {{- end }} 34 | ports: 35 | {{- if .Values.rabbitmqAmqpsSupport.enabled }} 36 | - name: amqps 37 | protocol: TCP 38 | port: {{ .Values.rabbitmqAmqpsSupport.amqpsNodePort }} 39 | targetPort: amqps 40 | {{- end }} 41 | {{ if and .Values.prometheus.exporter.enabled .Values.prometheus.exporter.exportOperator }} 42 | - name: exporter 43 | protocol: TCP 44 | port: {{ .Values.prometheus.exporter.port }} 45 | targetPort: exporter 46 | {{ end }} 47 | selector: 48 | app: {{ template "rabbitmq-ha.name" . }} 49 | release: {{ .Release.Name }} 50 | type: {{ .Values.service.type }} 51 | {{- end }} -------------------------------------------------------------------------------- /helm/templates/istio_serviceentry.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.istio.enabled }} 2 | {{- $fullName := (include "rabbitmq-ha.fullname" .) }} 3 | {{- $namespace := .Release.Namespace }} 4 | apiVersion: networking.istio.io/v1alpha3 5 | kind: ServiceEntry 6 | metadata: 7 | name: {{ template "rabbitmq-ha.fullname" . }} 8 | labels: 9 | app: {{ template "rabbitmq-ha.name" . }} 10 | chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} 11 | release: {{ .Release.Name }} 12 | heritage: {{ .Release.Service }} 13 | {{- if .Values.extraLabels }} 14 | {{ toYaml .Values.extraLabels | indent 4 }} 15 | {{- end }} 16 | spec: 17 | hosts: 18 | {{- range $counter, $e := untilStep 0 (.Values.replicaCount|int) 1 }} 19 | - {{ $fullName }}-{{ $counter }}.{{ $fullName }}-discovery.{{ $namespace }}.svc.cluster.local 20 | {{- end }} 21 | location: MESH_INTERNAL 22 | ports: 23 | - name: http 24 | protocol: TCP 25 | number: {{ .Values.rabbitmqManagerPort }} 26 | - name: amqp 27 | protocol: TCP 28 | number: {{ .Values.rabbitmqNodePort }} 29 | - name: epmd 30 | protocol: TCP 31 | number: {{ .Values.Epmd.rabbitmqEpmdPort }} 32 | {{- if .Values.rabbitmqSTOMPPlugin.enabled }} 33 | - name: stomp-tcp 34 | protocol: TCP 35 | number: 61613 36 | - name: stomp-ssl 37 | protocol: TCP 38 | number: 61614 39 | {{- end }} 40 | {{- if .Values.rabbitmqWebSTOMPPlugin.enabled }} 41 | - name: stomp-ws 42 | protocol: TCP 43 | number: 15674 44 | {{- end }} 45 | {{- if .Values.rabbitmqMQTTPlugin.enabled }} 46 | - name: mqtt-tcp 47 | protocol: TCP 48 | number: 1883 49 | - name: mqtt-ssl 50 | protocol: TCP 51 | number: 8883 52 | {{- end }} 53 | {{- if .Values.rabbitmqWebMQTTPlugin.enabled }} 54 | - name: mqtt-ws 55 | protocol: TCP 56 | number: 15675 57 | {{- end }} 58 | {{- if .Values.rabbitmqAmqpsSupport.enabled }} 59 | - name: amqps 60 | protocol: TCP 61 | number: {{ .Values.rabbitmqAmqpsSupport.amqpsNodePort }} 62 | {{- end }} 63 | {{ if .Values.prometheus.exporter.enabled }} 64 | - name: exporter 65 | protocol: TCP 66 | number: {{ .Values.prometheus.exporter.port }} 67 | {{ end }} 68 | - name: inter-node 69 | protocol: TCP 70 | number: 25672 71 | resolution: NONE 72 | {{- end }} -------------------------------------------------------------------------------- /helm/templates/service_internal.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: {{ template "rabbitmq-ha.fullname" . }}-internal 5 | labels: 6 | app: {{ template "rabbitmq-ha.name" . }} 7 | chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} 8 | release: {{ .Release.Name }} 9 | heritage: {{ .Release.Service }} 10 | {{- if .Values.extraLabels }} 11 | {{ toYaml .Values.extraLabels | indent 4 }} 12 | {{- end }} 13 | spec: 14 | ports: 15 | - name: http 16 | protocol: TCP 17 | port: {{ .Values.rabbitmqManagerPort }} 18 | targetPort: http 19 | - name: amqp 20 | protocol: TCP 21 | port: {{ .Values.rabbitmqNodePort }} 22 | targetPort: amqp 23 | {{- if .Values.Epmd.shouldExportEpmdPort }} 24 | - name: epmd 25 | protocol: TCP 26 | port: {{ .Values.Epmd.rabbitmqEpmdPort }} 27 | targetPort: epmd 28 | {{- end }} 29 | {{- if .Values.rabbitmqSTOMPPlugin.enabled }} 30 | - name: stomp-tcp 31 | protocol: TCP 32 | port: 61613 33 | targetPort: stomp-tcp 34 | - name: stomp-ssl 35 | protocol: TCP 36 | port: 61614 37 | targetPort: stomp-ssl 38 | {{- end }} 39 | {{- if .Values.rabbitmqWebSTOMPPlugin.enabled }} 40 | - name: stomp-ws 41 | protocol: TCP 42 | port: 15674 43 | targetPort: stomp-ws 44 | {{- end }} 45 | {{- if .Values.rabbitmqMQTTPlugin.enabled }} 46 | - name: mqtt-tcp 47 | protocol: TCP 48 | port: 1883 49 | targetPort: mqtt-tcp 50 | - name: mqtt-ssl 51 | protocol: TCP 52 | port: 8883 53 | targetPort: mqtt-ssl 54 | {{- end }} 55 | {{- if .Values.rabbitmqWebMQTTPlugin.enabled }} 56 | - name: mqtt-ws 57 | protocol: TCP 58 | port: 15675 59 | targetPort: mqtt-ws 60 | {{- end }} 61 | {{- if .Values.rabbitmqAmqpsSupport.enabled }} 62 | - name: amqps 63 | protocol: TCP 64 | port: {{ .Values.rabbitmqAmqpsSupport.amqpsNodePort }} 65 | targetPort: amqps 66 | {{- end }} 67 | {{ if .Values.prometheus.exporter.enabled }} 68 | - name: exporter 69 | protocol: TCP 70 | port: {{ .Values.prometheus.exporter.port }} 71 | targetPort: exporter 72 | {{ end }} 73 | selector: 74 | app: {{ template "rabbitmq-ha.name" . }} 75 | release: {{ .Release.Name }} 76 | -------------------------------------------------------------------------------- /helm/templates/service-discovery.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | annotations: 5 | service.alpha.kubernetes.io/tolerate-unready-endpoints: "true" 6 | {{- if .Values.service.annotations }} 7 | {{ toYaml .Values.service.annotations | indent 4 }} 8 | {{- end }} 9 | name: {{ template "rabbitmq-ha.fullname" . }}-discovery 10 | labels: 11 | app: {{ template "rabbitmq-ha.name" . }} 12 | chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} 13 | release: {{ .Release.Name }} 14 | heritage: {{ .Release.Service }} 15 | spec: 16 | clusterIP: None 17 | ports: 18 | - name: http 19 | protocol: TCP 20 | port: {{ .Values.rabbitmqManagerPort }} 21 | targetPort: http 22 | - name: amqp 23 | protocol: TCP 24 | port: {{ .Values.rabbitmqNodePort }} 25 | targetPort: amqp 26 | - name: epmd 27 | protocol: TCP 28 | port: {{ .Values.Epmd.rabbitmqEpmdPort }} 29 | targetPort: epmd 30 | {{- if .Values.rabbitmqSTOMPPlugin.enabled }} 31 | - name: stomp-tcp 32 | protocol: TCP 33 | port: 61613 34 | targetPort: stomp-tcp 35 | - name: stomp-ssl 36 | protocol: TCP 37 | port: 61614 38 | targetPort: stomp-ssl 39 | {{- end }} 40 | {{- if .Values.rabbitmqWebSTOMPPlugin.enabled }} 41 | - name: stomp-ws 42 | protocol: TCP 43 | port: 15674 44 | targetPort: stomp-ws 45 | {{- end }} 46 | {{- if .Values.rabbitmqMQTTPlugin.enabled }} 47 | - name: mqtt-tcp 48 | protocol: TCP 49 | port: 1883 50 | targetPort: mqtt-tcp 51 | - name: mqtt-ssl 52 | protocol: TCP 53 | port: 8883 54 | targetPort: mqtt-ssl 55 | {{- end }} 56 | {{- if .Values.rabbitmqWebMQTTPlugin.enabled }} 57 | - name: mqtt-ws 58 | protocol: TCP 59 | port: 15675 60 | targetPort: mqtt-ws 61 | {{- end }} 62 | {{- if .Values.rabbitmqAmqpsSupport.enabled }} 63 | - name: amqps 64 | protocol: TCP 65 | port: {{ .Values.rabbitmqAmqpsSupport.amqpsNodePort }} 66 | targetPort: amqps 67 | {{- end }} 68 | {{ if .Values.prometheus.exporter.enabled }} 69 | - name: exporter 70 | protocol: TCP 71 | port: {{ .Values.prometheus.exporter.port }} 72 | targetPort: exporter 73 | {{ end }} 74 | - name: inter-node 75 | protocol: TCP 76 | port: 25672 77 | targetPort: inter-node 78 | publishNotReadyAddresses: true 79 | selector: 80 | app: {{ template "rabbitmq-ha.name" . }} 81 | release: {{ .Release.Name }} 82 | type: ClusterIP 83 | -------------------------------------------------------------------------------- /helm/templates/istio_destrules.yaml: -------------------------------------------------------------------------------- 1 | {{- if and .Values.istio.enabled .Values.istio.mtls }} 2 | apiVersion: "networking.istio.io/v1alpha3" 3 | kind: DestinationRule 4 | metadata: 5 | name: {{ template "rabbitmq-ha.fullname" . }}-mtls-per-pod 6 | labels: 7 | app: {{ template "rabbitmq-ha.name" . }} 8 | chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} 9 | release: {{ .Release.Name }} 10 | heritage: {{ .Release.Service }} 11 | {{- if .Values.extraLabels }} 12 | {{ toYaml .Values.extraLabels | indent 4 }} 13 | {{- end }} 14 | spec: 15 | host: "*.{{ template "rabbitmq-ha.fullname" . }}-discovery.{{ .Release.Namespace }}.svc.cluster.local" 16 | trafficPolicy: 17 | tls: 18 | mode: ISTIO_MUTUAL 19 | --- 20 | apiVersion: "networking.istio.io/v1alpha3" 21 | kind: DestinationRule 22 | metadata: 23 | name: {{ template "rabbitmq-ha.fullname" . }}-mtls-discovery 24 | labels: 25 | app: {{ template "rabbitmq-ha.name" . }} 26 | chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} 27 | release: {{ .Release.Name }} 28 | heritage: {{ .Release.Service }} 29 | {{- if .Values.extraLabels }} 30 | {{ toYaml .Values.extraLabels | indent 4 }} 31 | {{- end }} 32 | spec: 33 | host: "{{ template "rabbitmq-ha.fullname" . }}-discovery.{{ .Release.Namespace }}.svc.cluster.local" 34 | trafficPolicy: 35 | tls: 36 | mode: ISTIO_MUTUAL 37 | portLevelSettings: #Disable client TLS requirements for EPMD port and optionally amqps port 38 | - port: 39 | number: {{ .Values.Epmd.rabbitmqEpmdPort }} 40 | tls: 41 | mode: DISABLE 42 | {{- if .Values.rabbitmqAmqpsSupport.enabled }} 43 | - port: 44 | number: {{ .Values.rabbitmqAmqpsSupport.amqpsNodePort }} 45 | tls: 46 | mode: DISABLE 47 | {{- end }} 48 | --- 49 | apiVersion: "networking.istio.io/v1alpha3" 50 | kind: DestinationRule 51 | metadata: 52 | name: {{ template "rabbitmq-ha.fullname" . }}-mtls 53 | labels: 54 | app: {{ template "rabbitmq-ha.name" . }} 55 | chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} 56 | release: {{ .Release.Name }} 57 | heritage: {{ .Release.Service }} 58 | {{- if .Values.extraLabels }} 59 | {{ toYaml .Values.extraLabels | indent 4 }} 60 | {{- end }} 61 | spec: 62 | host: "{{ template "rabbitmq-ha.fullname" . }}.{{ .Release.Namespace }}.svc.cluster.local" 63 | trafficPolicy: 64 | tls: 65 | mode: ISTIO_MUTUAL 66 | portLevelSettings: #Disable client TLS requirements for EPMD port and optionally amqps port 67 | {{- if .Values.rabbitmqAmqpsSupport.enabled }} 68 | - port: 69 | number: {{ .Values.rabbitmqAmqpsSupport.amqpsNodePort }} 70 | tls: 71 | mode: DISABLE 72 | {{- end }} 73 | --- 74 | apiVersion: "networking.istio.io/v1alpha3" 75 | kind: DestinationRule 76 | metadata: 77 | name: {{ template "rabbitmq-ha.fullname" . }}-mtls-internal 78 | labels: 79 | app: {{ template "rabbitmq-ha.name" . }} 80 | chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} 81 | release: {{ .Release.Name }} 82 | heritage: {{ .Release.Service }} 83 | {{- if .Values.extraLabels }} 84 | {{ toYaml .Values.extraLabels | indent 4 }} 85 | {{- end }} 86 | spec: 87 | host: "{{ template "rabbitmq-ha.fullname" . }}-internal.{{ .Release.Namespace }}.svc.cluster.local" 88 | trafficPolicy: 89 | tls: 90 | mode: ISTIO_MUTUAL 91 | portLevelSettings: #Disable client TLS requirements for EPMD port and optionally amqps port 92 | {{- if .Values.rabbitmqAmqpsSupport.enabled }} 93 | - port: 94 | number: {{ .Values.rabbitmqAmqpsSupport.amqpsNodePort }} 95 | tls: 96 | mode: DISABLE 97 | {{- end }} 98 | {{- end }} 99 | -------------------------------------------------------------------------------- /helm/templates/configmap.yaml: -------------------------------------------------------------------------------- 1 | {{- if not .Values.existingConfigMap }} 2 | apiVersion: v1 3 | kind: ConfigMap 4 | metadata: 5 | name: {{ template "rabbitmq-ha.fullname" . }} 6 | labels: 7 | app: {{ template "rabbitmq-ha.name" . }} 8 | chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} 9 | release: {{ .Release.Name }} 10 | heritage: {{ .Release.Service }} 11 | {{- if .Values.extraLabels }} 12 | {{ toYaml .Values.extraLabels | indent 4 }} 13 | {{- end }} 14 | data: 15 | enabled_plugins: | 16 | [ 17 | {{- if .Values.rabbitmqLDAPPlugin.enabled }} 18 | rabbitmq_auth_backend_ldap, 19 | {{- end }} 20 | 21 | {{- if .Values.rabbitmqMQTTPlugin.enabled }} 22 | rabbitmq_mqtt, 23 | {{- end }} 24 | 25 | {{- if .Values.rabbitmqWebMQTTPlugin.enabled }} 26 | rabbitmq_web_mqtt, 27 | {{- end }} 28 | 29 | {{- if .Values.rabbitmqSTOMPPlugin.enabled }} 30 | rabbitmq_stomp, 31 | {{- end }} 32 | 33 | {{- if .Values.rabbitmqWebSTOMPPlugin.enabled }} 34 | rabbitmq_web_stomp, 35 | {{- end }} 36 | 37 | {{- if .Values.rabbitmqAuth.enabled }} 38 | rabbitmq_auth_mechanism_ssl, 39 | {{- end }} 40 | 41 | {{- if .Values.rabbitmqAuthHTTP.enabled }} 42 | rabbitmq_auth_backend_http, 43 | {{- end }} 44 | 45 | rabbitmq_consistent_hash_exchange, 46 | rabbitmq_federation, 47 | rabbitmq_federation_management, 48 | rabbitmq_management, 49 | rabbitmq_peer_discovery_k8s, 50 | rabbitmq_shovel, 51 | rabbitmq_shovel_management 52 | ]. 53 | 54 | rabbitmq.conf: | 55 | ## RabbitMQ configuration 56 | ## Ref: https://github.com/rabbitmq/rabbitmq-server/blob/master/docs/rabbitmq.conf.example 57 | 58 | ## Authentification 59 | {{- if .Values.rabbitmqAuth.enabled }} 60 | {{ .Values.rabbitmqAuth.config | indent 4 }} 61 | {{- end }} 62 | 63 | ## Clustering 64 | cluster_formation.peer_discovery_backend = rabbit_peer_discovery_k8s 65 | cluster_formation.k8s.host = kubernetes.default.svc.cluster.local 66 | cluster_formation.k8s.address_type = hostname 67 | cluster_formation.node_cleanup.interval = 10 68 | # Set to false if automatic cleanup of absent nodes is desired. 69 | # This can be dangerous, see http://www.rabbitmq.com/cluster-formation.html#node-health-checks-and-cleanup. 70 | cluster_formation.node_cleanup.only_log_warning = true 71 | cluster_partition_handling = autoheal 72 | 73 | ## The default "guest" user is only permitted to access the server 74 | ## via a loopback interface (e.g. localhost) 75 | loopback_users.guest = false 76 | 77 | management.load_definitions = /etc/definitions/definitions.json 78 | 79 | ## Memory-based Flow Control threshold 80 | vm_memory_high_watermark.{{ .Values.rabbitmqMemoryHighWatermarkType }} = {{ .Values.rabbitmqMemoryHighWatermark }} 81 | 82 | ## Auth HTTP Backend Plugin 83 | {{- if .Values.rabbitmqAuthHTTP.enabled }} 84 | {{ .Values.rabbitmqAuthHTTP.config | indent 4 }} 85 | {{- end }} 86 | 87 | ## LDAP Plugin 88 | {{- if .Values.rabbitmqLDAPPlugin.enabled }} 89 | {{ .Values.rabbitmqLDAPPlugin.config | indent 4 }} 90 | {{- end }} 91 | 92 | ## MQTT Plugin 93 | {{- if .Values.rabbitmqMQTTPlugin.enabled }} 94 | {{ .Values.rabbitmqMQTTPlugin.config | indent 4 }} 95 | {{- end }} 96 | 97 | ## Web MQTT Plugin 98 | {{- if .Values.rabbitmqWebMQTTPlugin.enabled }} 99 | {{ .Values.rabbitmqWebMQTTPlugin.config | indent 4 }} 100 | {{- end }} 101 | 102 | ## STOMP Plugin 103 | {{- if .Values.rabbitmqSTOMPPlugin.enabled }} 104 | {{ .Values.rabbitmqSTOMPPlugin.config | indent 4 }} 105 | {{- end }} 106 | 107 | ## Web STOMP Plugin 108 | {{- if .Values.rabbitmqWebSTOMPPlugin.enabled }} 109 | {{ .Values.rabbitmqWebSTOMPPlugin.config | indent 4 }} 110 | {{- end }} 111 | 112 | ## AMQPS support 113 | {{- if .Values.rabbitmqAmqpsSupport.enabled }} 114 | {{ .Values.rabbitmqAmqpsSupport.config | indent 4 }} 115 | {{- end }} 116 | 117 | {{ .Values.extraConfig | indent 4 }} 118 | 119 | {{- end }} 120 | -------------------------------------------------------------------------------- /helm/templates/_helpers.tpl: -------------------------------------------------------------------------------- 1 | {{/* vim: set filetype=mustache: */}} 2 | {{/* 3 | Expand the name of the chart. 4 | */}} 5 | {{- define "rabbitmq-ha.name" -}} 6 | {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} 7 | {{- end -}} 8 | 9 | 10 | {{/* 11 | Helper template to loop json objects and create reoccuring objects 12 | */}} 13 | {{- define "rabbitmq-ha.duplicateJson" -}} 14 | {{- if and .Values.enabled .jsonCollection }}, 15 | {{- range $counter, $e := untilStep (.Values.startFrom|int) ((.Values.until|int)|add1|int) 1 }} 16 | {{- if (has ($e|toString) $.Values.skipIndexes | not) }} 17 | {{ $.jsonCollection | replace $.Values.replaceString ($e|toString) | indent 4}} 18 | {{- if lt $counter ( sub ((sub $.Values.until $.Values.startFrom)|add1|int) 1 ) -}} 19 | {{- printf "," -}} 20 | {{- end -}} 21 | {{- end -}} 22 | {{- end }} 23 | {{- end -}} 24 | {{- end -}} 25 | 26 | {{/* 27 | Create a default fully qualified app name. 28 | We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). 29 | If release name contains chart name it will be used as a full name. 30 | */}} 31 | {{- define "rabbitmq-ha.fullname" -}} 32 | {{- if .Values.fullnameOverride -}} 33 | {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} 34 | {{- else -}} 35 | {{- $name := default .Chart.Name .Values.nameOverride -}} 36 | {{- if contains $name .Release.Name -}} 37 | {{- .Release.Name | trunc 63 | trimSuffix "-" -}} 38 | {{- else -}} 39 | {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} 40 | {{- end -}} 41 | {{- end -}} 42 | {{- end -}} 43 | 44 | {{/* 45 | Create the name of the service account to use 46 | */}} 47 | {{- define "rabbitmq-ha.serviceAccountName" -}} 48 | {{- if .Values.serviceAccount.create -}} 49 | {{ default (include "rabbitmq-ha.fullname" .) .Values.serviceAccount.name }} 50 | {{- else -}} 51 | {{ default "default" .Values.serviceAccount.name }} 52 | {{- end -}} 53 | {{- end -}} 54 | 55 | {{/* 56 | Generate chart secret name 57 | */}} 58 | {{- define "rabbitmq-ha.secretName" -}} 59 | {{ default (include "rabbitmq-ha.fullname" .) .Values.existingSecret }} 60 | {{- end -}} 61 | 62 | {{/* 63 | Generate chart ssl secret name 64 | */}} 65 | {{- define "rabbitmq-ha.certSecretName" -}} 66 | {{ default (print (include "rabbitmq-ha.fullname" .) "-cert") .Values.rabbitmqCert.existingSecret }} 67 | {{- end -}} 68 | 69 | {{/* 70 | Defines a JSON file containing definitions of all broker objects (queues, exchanges, bindings, 71 | users, virtual hosts, permissions and parameters) to load by the management plugin. 72 | */}} 73 | {{- define "rabbitmq-ha.definitions" -}} 74 | { 75 | "users": [ 76 | { 77 | "name": {{ .Values.managementUsername | quote }}, 78 | "password": {{ .Values.managementPassword | quote }}, 79 | "tags": "management" 80 | }, 81 | { 82 | "name": {{ .Values.rabbitmqUsername | quote }}, 83 | "password": {{ .Values.rabbitmqPassword | quote }}, 84 | "tags": "administrator" 85 | }{{- if .Values.definitions.users -}}, 86 | {{ .Values.definitions.users | indent 4 }} 87 | {{- end }}{{include "rabbitmq-ha.duplicateJson" (dict "Values" .Values.definitions_reoccuring "jsonCollection" .Values.definitions_reoccuring.users)}} 88 | ], 89 | "vhosts": [ 90 | { 91 | "name": {{ .Values.rabbitmqVhost | quote }} 92 | }{{- if .Values.definitions.vhosts -}}, 93 | {{ .Values.definitions.vhosts | indent 4 }} 94 | {{- end }} 95 | ], 96 | "permissions": [ 97 | { 98 | "user": {{ .Values.rabbitmqUsername | quote }}, 99 | "vhost": {{ .Values.rabbitmqVhost | quote }}, 100 | "configure": ".*", 101 | "read": ".*", 102 | "write": ".*" 103 | }{{- if .Values.definitions.permissions -}}, 104 | {{ .Values.definitions.permissions | indent 4 }} 105 | {{- end }}{{include "rabbitmq-ha.duplicateJson" (dict "Values" .Values.definitions_reoccuring "jsonCollection" .Values.definitions_reoccuring.permissions)}} 106 | ], 107 | "parameters": [ 108 | {{ .Values.definitions.parameters| indent 4 }} 109 | ], 110 | "policies": [ 111 | {{ .Values.definitions.policies | indent 4 }} 112 | ], 113 | "queues": [ 114 | {{ .Values.definitions.queues | indent 4 }} 115 | ], 116 | "exchanges": [ 117 | {{ .Values.definitions.exchanges | indent 4 }} 118 | ], 119 | "bindings": [ 120 | {{ .Values.definitions.bindings| indent 4 }} 121 | ] 122 | } 123 | {{- end -}} 124 | -------------------------------------------------------------------------------- /helm/templates/NOTES.txt: -------------------------------------------------------------------------------- 1 | ** Please be patient while the chart is being deployed ** 2 | 3 | Credentials: 4 | 5 | Username : {{ .Values.rabbitmqUsername -}} 6 | {{ if .Values.existingSecret }} 7 | Password : $(kubectl get secret --namespace {{ .Release.Namespace }} {{ template "rabbitmq-ha.secretName" . }} -o jsonpath="{.data.rabbitmq-password}" | base64 --decode) 8 | ErLang Cookie : $(kubectl get secret --namespace {{ .Release.Namespace }} {{ template "rabbitmq-ha.secretName" . }} -o jsonpath="{.data.rabbitmq-erlang-cookie}" | base64 --decode) 9 | {{ else }} 10 | Password : $(kubectl get secret --namespace {{ .Release.Namespace }} {{ template "rabbitmq-ha.fullname" . }} -o jsonpath="{.data.rabbitmq-password}" | base64 --decode) 11 | ErLang Cookie : $(kubectl get secret --namespace {{ .Release.Namespace }} {{ template "rabbitmq-ha.fullname" . }} -o jsonpath="{.data.rabbitmq-erlang-cookie}" | base64 --decode) 12 | {{ end }} 13 | 14 | RabbitMQ can be accessed within the cluster on port {{ .Values.rabbitmqNodePort }} at {{ template "rabbitmq-ha.fullname" . }}.{{ .Release.Namespace }}.svc.cluster.local 15 | 16 | To access the cluster externally execute the following commands: 17 | 18 | {{- if contains "NodePort" .Values.service.type }} 19 | 20 | export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[?(@.type=='ExternalIP')].address}") 21 | export NODE_PORT_AMQP=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath='{.spec.ports[?(@.name=="amqp")].nodePort}' services {{ template "rabbitmq-ha.fullname" . }}) 22 | {{- if .Values.rabbitmqAmqpsSupport.enabled }} 23 | export NODE_PORT_AMQPS=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath='{.spec.ports[?(@.name=="amqps")].nodePort}' services {{ template "rabbitmq-ha.fullname" . }}) 24 | {{- end }} 25 | export NODE_PORT_STATS=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath='{.spec.ports[?(@.name=="http")].nodePort}' services {{ template "rabbitmq-ha.fullname" . }}) 26 | {{- if .Values.prometheus.exporter.enabled }} 27 | export NODE_PORT_RABBITMQ_EXPORTER=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath='{.spec.ports[?(@.name=="exporter")].nodePort}' services {{ template "rabbitmq-ha.fullname" . }}) 28 | {{- end }} 29 | To Access the RabbitMQ AMQP port: 30 | 31 | amqp://$NODE_IP:$NODE_PORT_AMQP/ 32 | 33 | {{- if .Values.rabbitmqAmqpsSupport.enabled }} 34 | To Access the RabbitMQ AMQPS port: 35 | 36 | amqps://$NODE_IP:$NODE_PORT_AMQPS/ 37 | {{- end }} 38 | 39 | {{- if .Values.prometheus.exporter.enabled }} 40 | To Access the RabbitMQ Exporter metrics port: 41 | http://$NODE_IP:$NODE_PORT_RABBITMQ_EXPORTER/ 42 | {{- end }} 43 | 44 | To Access the RabbitMQ Management interface: 45 | 46 | http://$NODE_IP:$NODE_PORT_STATS/ 47 | 48 | {{- else if contains "LoadBalancer" .Values.service.type }} 49 | 50 | NOTE: It may take a few minutes for the LoadBalancer IP to be available. 51 | Watch the status with: 'kubectl get svc --namespace {{ .Release.Namespace }} -w {{ template "rabbitmq-ha.name" . }}' 52 | 53 | export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ template "rabbitmq-ha.fullname" . }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}') 54 | {{- if .Values.rabbitmqAmqpsSupport.enabled }} 55 | export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ template "rabbitmq-ha.fullname" . }} -o jsonpath='{.status.loadBalancer.ingress[1].ip}') 56 | {{- end }} 57 | 58 | To Access the RabbitMQ AMQP port: 59 | 60 | amqp://$SERVICE_IP:{{ .Values.rabbitmqNodePort }}/ 61 | 62 | {{- if .Values.rabbitmqAmqpsSupport.enabled }} 63 | To Access the RabbitMQ AMQPS port: 64 | 65 | amqps://$SERVICE_IP:{{ .Values.rabbitmqAmqpsSupport.amqpsNodePort }}/ 66 | {{- end }} 67 | 68 | {{- if .Values.prometheus.exporter.enabled }} 69 | To Access the RabbitMQ Exporter metrics port: 70 | 71 | http://$SERVICE_IP:{{ .Values.prometheus.exporter.port }}/ 72 | {{- end }} 73 | 74 | To Access the RabbitMQ Management interface: 75 | 76 | http://$SERVICE_IP:{{ .Values.rabbitmqManagerPort }}/ 77 | 78 | {{- else if contains "ClusterIP" .Values.service.type }} 79 | 80 | export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app={{ template "rabbitmq-ha.name" . }}" -o jsonpath="{.items[0].metadata.name}") 81 | kubectl port-forward $POD_NAME --namespace {{ .Release.Namespace }} {{ .Values.rabbitmqNodePort }}:{{ .Values.rabbitmqNodePort }} {{ .Values.rabbitmqManagerPort }}:{{ .Values.rabbitmqManagerPort }} 82 | {{- if .Values.rabbitmqAmqpsSupport.enabled }} 83 | kubectl port-forward $POD_NAME --namespace {{ .Release.Namespace }} {{ .Values.rabbitmqAmqpsSupport.amqpsNodePort }}:{{ .Values.rabbitmqAmqpsSupport.amqpsNodePort }} 84 | {{- end }} 85 | 86 | {{- if .Values.prometheus.exporter.enabled }} 87 | kubectl port-forward $POD_NAME --namespace {{ .Release.Namespace }} {{ .Values.prometheus.exporter.port }}:{{ .Values.prometheus.exporter.port }} 88 | {{- end }} 89 | 90 | To Access the RabbitMQ AMQP port: 91 | 92 | amqp://127.0.0.1:{{ .Values.rabbitmqNodePort }}/ 93 | 94 | {{- if .Values.rabbitmqAmqpsSupport.enabled }} 95 | To Access the RabbitMQ AMQPS port: 96 | 97 | amqps://127.0.0.1:{{ .Values.rabbitmqAmqpsSupport.amqpsNodePort }}/ 98 | {{- end }} 99 | 100 | {{- if .Values.prometheus.exporter.enabled }} 101 | To Access the RabbitMQ Exporter metrics port: 102 | 103 | http://127.0.0.1:{{ .Values.prometheus.exporter.port }}/ 104 | {{- end }} 105 | 106 | To Access the RabbitMQ Management interface: 107 | 108 | URL : http://127.0.0.1:{{ .Values.rabbitmqManagerPort }} 109 | {{- end }} 110 | 111 | -------------------------------------------------------------------------------- /helm/templates/alerts.yaml: -------------------------------------------------------------------------------- 1 | {{ if and .Values.prometheus.exporter.enabled .Values.prometheus.operator.enabled .Values.prometheus.operator.alerts.enabled }} 2 | apiVersion: monitoring.coreos.com/v1 3 | kind: PrometheusRule 4 | metadata: 5 | name: {{ .Release.Name }}-rabbitmq-alerts 6 | labels: 7 | app: "rabbitmq" 8 | chart: {{ .Chart.Name }}-{{ .Chart.Version }} 9 | heritage: {{ .Release.Service }} 10 | release: {{ .Release.Name }} 11 | {{- if .Values.prometheus.operator.serviceMonitor.selector }} 12 | {{ toYaml .Values.prometheus.operator.serviceMonitor.selector | indent 4 }} 13 | {{- end }} 14 | {{- if .Values.prometheus.operator.alerts.selector }} 15 | {{ toYaml .Values.prometheus.operator.alerts.selector | indent 4 }} 16 | {{- end }} 17 | spec: 18 | groups: 19 | - name: rabbitmq-alerts 20 | rules: 21 | - alert: RabbitMqClusterNodeDown 22 | expr: rabbitmq_up{service="{{ template "rabbitmq-ha.fullname" . }}-discovery"} == 0 23 | for: 5m 24 | labels: 25 | installed_by: {{ .Release.Name }} 26 | severity: critical 27 | {{- if .Values.prometheus.operator.alerts.labels }} 28 | {{ toYaml .Values.prometheus.operator.alerts.labels | indent 8 }} 29 | {{- end }} 30 | annotations: 31 | description: RabbitMQ {{`{{ $labels.namespace }}`}}/{{`{{ $labels.pod}}`}} is down 32 | summary: RabbitMQ Node Is Down 33 | - alert: RabbitMQExporterNotAvailable 34 | expr: absent(rabbitmq_up) 35 | labels: 36 | installed_by: {{ .Release.Name }} 37 | severity: critical 38 | {{- if .Values.prometheus.operator.alerts.labels }} 39 | {{ toYaml .Values.prometheus.operator.alerts.labels | indent 8 }} 40 | {{- end }} 41 | annotations: 42 | description: 'RabbitMQ Exporter Instance "{{`{{ $labels.instance }}`}}" not found to report rabbitmq status.' 43 | summary: RabbitMQ Exporter Instance is down 44 | - alert: RabbitMqClusterNotAllNodesRunning 45 | expr: sum(rabbitmq_up{service="{{ template "rabbitmq-ha.fullname" . }}-discovery"}) by (service) < {{ .Values.replicaCount }} 46 | for: 5m 47 | labels: 48 | installed_by: {{ .Release.Name }} 49 | severity: critical 50 | team: devops 51 | annotations: 52 | description: Some RabbitMQ Cluster Nodes Are Down in Service {{`{{ $labels.namespace }}`}}/{{`{{ $labels.service}}`}} 53 | summary: Some RabbitMQ Cluster Nodes Are Down in Service {{`{{ $labels.namespace }}`}}/{{`{{ $labels.service}}`}} 54 | - alert: RabbitMqDiskSpaceAlarm 55 | expr: rabbitmq_node_disk_free_alarm{service="{{ template "rabbitmq-ha.fullname" . }}-discovery"} == 1 56 | for: 1m 57 | labels: 58 | installed_by: {{ .Release.Name }} 59 | severity: critical 60 | {{- if .Values.prometheus.operator.alerts.labels }} 61 | {{ toYaml .Values.prometheus.operator.alerts.labels | indent 8 }} 62 | {{- end }} 63 | annotations: 64 | description: RabbitMQ {{`{{ $labels.namespace }}`}}/{{`{{ $labels.pod}}`}} Disk Space Alarm is going off. Which means the node hit highwater mark and has cut off network connectivity, see RabbitMQ WebUI 65 | summary: RabbitMQ is Out of Disk Space 66 | - alert: RabbitMqMemoryAlarm 67 | expr: rabbitmq_node_mem_alarm{service="{{ template "rabbitmq-ha.fullname" . }}-discovery"} == 1 68 | for: 1m 69 | labels: 70 | installed_by: {{ .Release.Name }} 71 | severity: critical 72 | {{- if .Values.prometheus.operator.alerts.labels }} 73 | {{ toYaml .Values.prometheus.operator.alerts.labels | indent 8 }} 74 | {{- end }} 75 | annotations: 76 | description: RabbitMQ {{`{{ $labels.namespace }}`}}/{{`{{ $labels.pod}}`}} High Memory Alarm is going off. Which means the node hit highwater mark and has cut off network connectivity, see RabbitMQ WebUI 77 | summary: RabbitMQ is Out of Memory 78 | - alert: RabbitMqMemoryUsageHigh 79 | expr: (rabbitmq_node_mem_used{service="{{ template "rabbitmq-ha.fullname" . }}-discovery"} / rabbitmq_node_mem_limit{service="{{ template "rabbitmq-ha.fullname" . }}-discovery"}) > .9 80 | for: 1m 81 | labels: 82 | installed_by: {{ .Release.Name }} 83 | severity: critical 84 | {{- if .Values.prometheus.operator.alerts.labels }} 85 | {{ toYaml .Values.prometheus.operator.alerts.labels | indent 8 }} 86 | {{- end }} 87 | annotations: 88 | description: RabbitMQ {{`{{ $labels.namespace }}`}}/{{`{{ $labels.pod}}`}} Memory Usage > 90% 89 | summary: RabbitMQ Node > 90% Memory Usage 90 | - alert: RabbitMqFileDescriptorsLow 91 | expr: (rabbitmq_fd_used{service="{{ template "rabbitmq-ha.fullname" . }}-discovery"} / rabbitmq_fd_total{service="{{ template "rabbitmq-ha.fullname" . }}-discovery"}) > .9 92 | for: 5m 93 | labels: 94 | installed_by: {{ .Release.Name }} 95 | severity: critical 96 | {{- if .Values.prometheus.operator.alerts.labels }} 97 | {{ toYaml .Values.prometheus.operator.alerts.labels | indent 8 }} 98 | {{- end }} 99 | annotations: 100 | description: RabbitMQ {{`{{ $labels.namespace }}`}}/{{`{{ $labels.pod}}`}} File Descriptor Usage > 90% 101 | summary: RabbitMQ Low File Descriptor Available 102 | - alert: RabbitMqDiskSpaceLow 103 | expr: predict_linear(rabbitmq_node_disk_free{service="{{ template "rabbitmq-ha.fullname" . }}-discovery"}[15m], 1 * 60 * 60) < rabbitmq_node_disk_free_limit{service="{{ template "rabbitmq-ha.fullname" . }}-discovery"} 104 | for: 5m 105 | labels: 106 | installed_by: {{ .Release.Name }} 107 | severity: critical 108 | {{- if .Values.prometheus.operator.alerts.labels }} 109 | {{ toYaml .Values.prometheus.operator.alerts.labels | indent 8 }} 110 | {{- end }} 111 | annotations: 112 | description: RabbitMQ {{`{{ $labels.namespace }}`}}/{{`{{ $labels.pod}}`}} will hit disk limit in the next hr based on last 15 mins trend. 113 | summary: RabbitMQ is Low on Disk Space and will Run Out in the next hour 114 | {{ end }} 115 | -------------------------------------------------------------------------------- /helm/templates/statefulset.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1beta1 2 | kind: StatefulSet 3 | metadata: 4 | name: {{ template "rabbitmq-ha.fullname" . }} 5 | labels: 6 | app: {{ template "rabbitmq-ha.name" . }} 7 | chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} 8 | release: {{ .Release.Name }} 9 | heritage: {{ .Release.Service }} 10 | {{- if .Values.extraLabels }} 11 | {{ toYaml .Values.extraLabels | indent 4 }} 12 | {{- end }} 13 | spec: 14 | podManagementPolicy: {{ .Values.podManagementPolicy }} 15 | serviceName: {{ template "rabbitmq-ha.fullname" . }}-discovery 16 | replicas: {{ .Values.replicaCount }} 17 | updateStrategy: 18 | type: {{ .Values.updateStrategy }} 19 | template: 20 | metadata: 21 | labels: 22 | app: {{ template "rabbitmq-ha.name" . }} 23 | release: {{ .Release.Name }} 24 | {{- if .Values.extraLabels }} 25 | {{ toYaml .Values.extraLabels | indent 8 }} 26 | {{- end }} 27 | annotations: 28 | {{- if not .Values.existingConfigMap }} 29 | checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} 30 | {{- end }} 31 | {{- if and .Values.prometheus.exporter.enabled (not .Values.prometheus.operator.enabled) }} 32 | prometheus.io/scrape: "true" 33 | prometheus.io/port: {{ .Values.prometheus.exporter.port | quote }} 34 | {{- end }} 35 | {{- if .Values.podAnnotations }} 36 | {{ toYaml .Values.podAnnotations | indent 8 }} 37 | {{- end }} 38 | spec: 39 | {{- if .Values.image.pullSecrets }} 40 | imagePullSecrets: 41 | {{- range .Values.image.pullSecrets }} 42 | - name: {{ . }} 43 | {{- end }} 44 | {{- end }} 45 | terminationGracePeriodSeconds: {{ .Values.terminationGracePeriodSeconds }} 46 | serviceAccountName: {{ template "rabbitmq-ha.serviceAccountName" . }} 47 | initContainers: 48 | - name: copy-rabbitmq-config 49 | image: {{ .Values.image.busyboxImage }} 50 | command: ['sh', '-c', 'cp /configmap/* /etc/rabbitmq; rm -f /var/lib/rabbitmq/.erlang.cookie'] 51 | resources: 52 | {{ toYaml .Values.initContainer.resources | indent 12 }} 53 | volumeMounts: 54 | - name: configmap 55 | mountPath: /configmap 56 | - name: config 57 | mountPath: /etc/rabbitmq 58 | - name: {{ .Values.persistentVolume.name }} 59 | mountPath: /var/lib/rabbitmq 60 | containers: 61 | - name: {{ .Chart.Name }} 62 | image: {{ .Values.image.repository }}:{{ .Values.image.tag }} 63 | imagePullPolicy: {{ .Values.image.pullPolicy }} 64 | ports: 65 | - name: epmd 66 | protocol: TCP 67 | containerPort: 4369 68 | - name: amqp 69 | protocol: TCP 70 | containerPort: 5672 71 | - name: http 72 | protocol: TCP 73 | containerPort: 15672 74 | {{- if .Values.rabbitmqSTOMPPlugin.enabled }} 75 | - name: stomp-tcp 76 | protocol: TCP 77 | containerPort: 61613 78 | - name: stomp-ssl 79 | protocol: TCP 80 | containerPort: 61614 81 | {{- end }} 82 | {{- if .Values.rabbitmqWebSTOMPPlugin.enabled }} 83 | - name: stomp-ws 84 | protocol: TCP 85 | containerPort: 15674 86 | {{- end }} 87 | {{- if .Values.rabbitmqMQTTPlugin.enabled }} 88 | - name: mqtt-tcp 89 | protocol: TCP 90 | containerPort: 1883 91 | - name: mqtt-ssl 92 | protocol: TCP 93 | containerPort: 8883 94 | {{- end }} 95 | {{- if .Values.rabbitmqWebMQTTPlugin.enabled }} 96 | - name: mqtt-ws 97 | protocol: TCP 98 | containerPort: 15675 99 | {{- end }} 100 | {{- if .Values.rabbitmqAmqpsSupport.enabled }} 101 | - name: amqps 102 | protocol: TCP 103 | containerPort: 5671 104 | {{- end }} 105 | - name: inter-node 106 | protocol: TCP 107 | containerPort: 25672 108 | {{- $managementCredentials := printf "%s:%s" .Values.managementUsername .Values.managementPassword | b64enc }} 109 | {{- $managementHeader := printf "Authorization: Basic %s" $managementCredentials }} 110 | livenessProbe: 111 | exec: 112 | command: 113 | - /bin/sh 114 | - -c 115 | - 'wget -O - -q --header "{{ $managementHeader }}" http://localhost:15672/api/healthchecks/node | grep -qF "{\"status\":\"ok\"}"' 116 | initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} 117 | periodSeconds: {{ .Values.livenessProbe.periodSeconds }} 118 | timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }} 119 | failureThreshold: {{ .Values.livenessProbe.failureThreshold }} 120 | readinessProbe: 121 | exec: 122 | command: 123 | - /bin/sh 124 | - -c 125 | - 'wget -O - -q --header "{{ $managementHeader }}" http://localhost:15672/api/healthchecks/node | grep -qF "{\"status\":\"ok\"}"' 126 | initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} 127 | timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }} 128 | periodSeconds: {{ .Values.readinessProbe.periodSeconds }} 129 | failureThreshold: {{ .Values.readinessProbe.failureThreshold }} 130 | env: 131 | - name: MY_POD_NAME 132 | valueFrom: 133 | fieldRef: 134 | apiVersion: v1 135 | fieldPath: metadata.name 136 | - name: RABBITMQ_USE_LONGNAME 137 | value: "true" 138 | - name: RABBITMQ_NODENAME 139 | value: rabbit@$(MY_POD_NAME).{{ template "rabbitmq-ha.fullname" . }}-discovery.{{ .Release.Namespace }}.svc.cluster.local 140 | - name: K8S_HOSTNAME_SUFFIX 141 | value: .{{ template "rabbitmq-ha.fullname" . }}-discovery.{{ .Release.Namespace }}.svc.cluster.local 142 | - name: K8S_SERVICE_NAME 143 | value: {{ template "rabbitmq-ha.fullname" . }}-discovery 144 | - name: RABBITMQ_ERLANG_COOKIE 145 | valueFrom: 146 | secretKeyRef: 147 | name: {{ template "rabbitmq-ha.secretName" . }} 148 | key: rabbitmq-erlang-cookie 149 | {{- if .Values.rabbitmqHipeCompile }} 150 | - name: RABBITMQ_HIPE_COMPILE 151 | value: {{ .Values.rabbitmqHipeCompile | quote }} 152 | {{- end }} 153 | resources: 154 | {{ toYaml .Values.resources | indent 12 }} 155 | volumeMounts: 156 | - name: {{ .Values.persistentVolume.name }} 157 | mountPath: /var/lib/rabbitmq 158 | - name: config 159 | mountPath: /etc/rabbitmq 160 | readOnly: true 161 | {{- if not .Values.existingSecret }} 162 | - name: definitions 163 | mountPath: /etc/definitions 164 | readOnly: true 165 | {{- end }} 166 | {{- if .Values.rabbitmqCert.enabled }} 167 | - name: cert 168 | mountPath: /etc/cert 169 | {{- end }} 170 | {{ if .Values.prometheus.exporter.enabled }} 171 | - name: {{ .Chart.Name }}-exporter 172 | image: {{ .Values.prometheus.exporter.image.repository }}:{{ .Values.prometheus.exporter.image.tag }} 173 | imagePullPolicy: {{ .Values.prometheus.exporter.image.pullPolicy }} 174 | ports: 175 | - name: exporter 176 | protocol: TCP 177 | containerPort: {{ .Values.prometheus.exporter.port }} 178 | env: 179 | - name: PUBLISH_PORT 180 | value: "{{ .Values.prometheus.exporter.port }}" 181 | {{ if .Values.prometheus.exporter.capabilities }} 182 | - name: RABBIT_CAPABILITIES 183 | value: "{{ .Values.prometheus.exporter.capabilities }}" 184 | {{- end }} 185 | - name: RABBIT_USER 186 | valueFrom: 187 | secretKeyRef: 188 | name: {{ template "rabbitmq-ha.secretName" . }} 189 | key: rabbitmq-username 190 | - name: RABBIT_PASSWORD 191 | valueFrom: 192 | secretKeyRef: 193 | name: {{ template "rabbitmq-ha.secretName" . }} 194 | key: rabbitmq-password 195 | {{- range $key, $value := .Values.prometheus.exporter.env }} 196 | - name: {{ $key }} 197 | value: {{ $value | quote }} 198 | {{- end }} 199 | resources: 200 | {{ toYaml .Values.prometheus.exporter.resources | indent 12 }} 201 | {{ end }} 202 | {{- if .Values.nodeSelector }} 203 | nodeSelector: 204 | {{ toYaml .Values.nodeSelector | indent 8 }} 205 | {{- end }} 206 | {{- if .Values.tolerations }} 207 | tolerations: 208 | {{ toYaml .Values.tolerations | indent 8 }} 209 | {{- end }} 210 | {{- if eq .Values.podAntiAffinity "hard" }} 211 | affinity: 212 | podAntiAffinity: 213 | requiredDuringSchedulingIgnoredDuringExecution: 214 | - topologyKey: "kubernetes.io/hostname" 215 | labelSelector: 216 | matchLabels: 217 | app: {{ template "rabbitmq-ha.name" . }} 218 | release: {{ .Release.Name }} 219 | {{- else if eq .Values.podAntiAffinity "soft" }} 220 | affinity: 221 | podAntiAffinity: 222 | preferredDuringSchedulingIgnoredDuringExecution: 223 | - weight: 1 224 | podAffinityTerm: 225 | topologyKey: kubernetes.io/hostname 226 | labelSelector: 227 | matchLabels: 228 | app: {{ template "rabbitmq-ha.name" . }} 229 | release: {{ .Release.Name }} 230 | {{- end }} 231 | volumes: 232 | - name: config 233 | emptyDir: {} 234 | - name: configmap 235 | configMap: 236 | name: {{ template "rabbitmq-ha.fullname" . }} 237 | {{- if not .Values.existingSecret }} 238 | - name: definitions 239 | secret: 240 | secretName: {{ template "rabbitmq-ha.fullname" . }} 241 | items: 242 | - key: definitions.json 243 | path: definitions.json 244 | {{- end }} 245 | {{- if .Values.rabbitmqCert.enabled }} 246 | - name: cert 247 | secret: 248 | defaultMode: 420 249 | secretName: {{ template "rabbitmq-ha.certSecretName" . }} 250 | {{- end }} 251 | {{- if .Values.priorityClassName }} 252 | priorityClassName: {{ .Values.priorityClassName }} 253 | {{- end }} 254 | {{- if .Values.persistentVolume.enabled }} 255 | volumeClaimTemplates: 256 | - metadata: 257 | name: {{ .Values.persistentVolume.name }} 258 | annotations: 259 | {{- range $key, $value := .Values.persistentVolume.annotations }} 260 | {{ $key }}: {{ $value }} 261 | {{- end }} 262 | spec: 263 | accessModes: 264 | {{- range .Values.persistentVolume.accessModes }} 265 | - {{ . | quote }} 266 | {{- end }} 267 | resources: 268 | requests: 269 | storage: {{ .Values.persistentVolume.size | quote }} 270 | {{- if .Values.persistentVolume.storageClass }} 271 | {{- if (eq "-" .Values.persistentVolume.storageClass) }} 272 | storageClassName: "" 273 | {{- else }} 274 | storageClassName: "{{ .Values.persistentVolume.storageClass }}" 275 | {{- end }} 276 | {{- end }} 277 | {{- else }} 278 | - name: data 279 | emptyDir: {} 280 | {{- end }} 281 | -------------------------------------------------------------------------------- /helm/values.yaml: -------------------------------------------------------------------------------- 1 | ## RabbitMQ application credentials 2 | ## Ref: http://rabbitmq.com/access-control.html 3 | ## 4 | rabbitmqUsername: adminuser 5 | rabbitmqPassword: password 6 | 7 | ## RabbitMQ Management user used for health checks 8 | managementUsername: management 9 | managementPassword: management 10 | fullnameOverride: rabbitmq 11 | 12 | # Important note - the MTLS policy is deployed for the entire namespace, and it's called "Default". 13 | # From documentation, there can only be one per namespace policy - meaning, it's advised to install rabbitMQ in a seperate namespace so the policy will apply to this deployment. 14 | # https://istio.io/docs/concepts/security/#target-selectors - Namespace-wide policy: A policy defined in the namespace-scope storage with name default and no target selector section. There can be at most one namespace-wide policy per namespace 15 | istio: 16 | enabled: true 17 | mtls: true 18 | ingress: 19 | enabled: false 20 | managementHostName: rabbitui.mydomain.com 21 | 22 | ## Place any additional key/value configuration to add to rabbitmq.conf 23 | ## Ref: https://www.rabbitmq.com/configure.html#config-items 24 | extraConfig: | 25 | # log.console.level = debug 26 | 27 | ## Place any reoccurring values here - to create multiple values in the definitions.json 28 | definitions_reoccuring: 29 | enabled: true 30 | replaceString: "#" 31 | startFrom: 100 32 | until: 120 33 | ## Must be string values ( e.g. ["102", "103"] 34 | skipIndexes: ["110"] 35 | users: |- 36 | { 37 | "name": "user#", 38 | "tags": "" 39 | } 40 | permissions: |- 41 | { 42 | "user": "user#", 43 | "vhost": "/", 44 | "configure": ".*", 45 | "write": ".*", 46 | "read": ".*" 47 | } 48 | ## TODO: Change the password of the user as it connects with client certificate 49 | definitions: 50 | users: |- 51 | { 52 | "name": "singleuser", 53 | "tags": "", 54 | "password": "password" 55 | } 56 | vhosts: |- 57 | { 58 | "name": "/" 59 | } 60 | parameters: |- 61 | # { 62 | # "value": { 63 | # "src-uri": "amqp://localhost", 64 | # "src-queue": "source", 65 | # "dest-uri": "amqp://localhost", 66 | # "dest-queue": "destination", 67 | # "add-forward-headers": false, 68 | # "ack-mode": "on-confirm", 69 | # "delete-after": "never" 70 | # }, 71 | # "vhost": "/", 72 | # "component": "shovel", 73 | # "name": "test" 74 | # } 75 | permissions: |- 76 | { 77 | "user": "singleuser", 78 | "vhost": "/", 79 | "configure": ".*", 80 | "write": ".*", 81 | "read": ".*" 82 | }, 83 | { 84 | "user": "adminuser", 85 | "vhost": "/", 86 | "configure": ".*", 87 | "write": ".*", 88 | "read": ".*" 89 | } 90 | queues: |- 91 | # { 92 | # "name":"myName", 93 | # "vhost":"/rabbit", 94 | # "durable":true, 95 | # "auto_delete":false, 96 | # "arguments":{} 97 | # } 98 | exchanges: |- 99 | # { 100 | # "name":"myName", 101 | # "vhost":"/rabbit", 102 | # "type":"direct", 103 | # "durable":true, 104 | # "auto_delete":false, 105 | # "internal":false, 106 | # "arguments":{} 107 | # } 108 | bindings: |- 109 | # { 110 | # "source":"myName", 111 | # "vhost":"/rabbit", 112 | # "destination":"myName", 113 | # "destination_type":"queue", 114 | # "routing_key":"myKey", 115 | # "arguments":{} 116 | # } 117 | ## Sets the policies in definitions.json. This can be used to control the high 118 | ## availability of queues by mirroring them to multiple nodes. 119 | ## Ref: https://www.rabbitmq.com/ha.html 120 | policies: |- 121 | # { 122 | # "name": "ha-ti", 123 | # "pattern": "^somepolicy.*", 124 | # "vhost": "/", 125 | # "definition": { 126 | # "ha-mode": "all", 127 | # "ha-sync-mode": "manual", 128 | # "ha-promote-on-shutdown": "always", 129 | # "ha-promote-on-failure": "always" 130 | # } 131 | # } 132 | ## RabbitMQ default VirtualHost 133 | ## Ref: https://www.rabbitmq.com/vhosts.html 134 | ## 135 | rabbitmqVhost: "/" 136 | 137 | ## Erlang cookie to determine whether different nodes are allowed to communicate with each other 138 | ## Ref: https://www.rabbitmq.com/clustering.html 139 | ## sensitive - please switch with a random alphanumeric [a-zA-Z] up to 255 chars value 140 | rabbitmqErlangCookie: cookie 141 | 142 | ## RabbitMQ Memory high watermark 143 | ## Ref: http://www.rabbitmq.com/memory.html 144 | ## 145 | rabbitmqMemoryHighWatermark: 1024MB 146 | rabbitmqMemoryHighWatermarkType: absolute 147 | 148 | ## EPMD port for peer discovery service used by RabbitMQ nodes and CLI tools 149 | ## Ref: https://www.rabbitmq.com/clustering.html 150 | ## 151 | Epmd: 152 | shouldExportEpmdPort: false 153 | rabbitmqEpmdPort: 4369 154 | 155 | ## Node port 156 | rabbitmqNodePort: 5672 157 | 158 | ## Manager port 159 | rabbitmqManagerPort: 15672 160 | 161 | ## Set to true to precompile parts of RabbitMQ with HiPE, a just-in-time 162 | ## compiler for Erlang. This will increase server throughput at the cost of 163 | ## increased startup time. You might see 20-50% better performance at the cost 164 | ## of a few minutes delay at startup. 165 | rabbitmqHipeCompile: false 166 | 167 | ## SSL certificates 168 | ## Red: http://www.rabbitmq.com/ssl.html 169 | rabbitmqCert: 170 | enabled: false 171 | 172 | # Specifies an existing secret to be used for SSL Certs 173 | existingSecret: "" 174 | 175 | ## Create a new secret using these values 176 | cacertfile: | 177 | certfile: | 178 | keyfile: | 179 | 180 | ## Authentication mechanism 181 | ## Ref: http://www.rabbitmq.com/authentication.html 182 | rabbitmqAuth: 183 | enabled: false 184 | 185 | config: | 186 | # auth_mechanisms.1 = PLAIN 187 | # auth_mechanisms.2 = EXTERNAL 188 | # ssl_cert_login_from = common_name 189 | # auth_mechanisms.2 = AMQPLAIN 190 | 191 | ## Authentication backend 192 | ## Ref: https://github.com/rabbitmq/rabbitmq-auth-backend-http 193 | rabbitmqAuthHTTP: 194 | enabled: false 195 | 196 | config: | 197 | # auth_backends.1 = http 198 | # auth_http.user_path = http://some-server/auth/user 199 | # auth_http.vhost_path = http://some-server/auth/vhost 200 | # auth_http.resource_path = http://some-server/auth/resource 201 | # auth_http.topic_path = http://some-server/auth/topic 202 | 203 | ## LDAP Plugin 204 | ## Ref: http://www.rabbitmq.com/ldap.html 205 | rabbitmqLDAPPlugin: 206 | enabled: false 207 | 208 | ## LDAP configuration: 209 | config: | 210 | # auth_backends.1 = ldap 211 | # auth_ldap.servers.1 = my-ldap-server 212 | # auth_ldap.user_dn_pattern = cn=${username},ou=People,dc=example,dc=com 213 | # auth_ldap.use_ssl = false 214 | # auth_ldap.port = 389 215 | # auth_ldap.log = false 216 | 217 | ## MQTT Plugin 218 | ## Ref: http://www.rabbitmq.com/mqtt.html 219 | rabbitmqMQTTPlugin: 220 | enabled: false 221 | 222 | ## MQTT configuration: 223 | config: | 224 | # mqtt.default_user = guest 225 | # mqtt.default_pass = guest 226 | # mqtt.allow_anonymous = true 227 | 228 | ## Web MQTT Plugin 229 | ## Ref: http://www.rabbitmq.com/web-mqtt.html 230 | rabbitmqWebMQTTPlugin: 231 | enabled: false 232 | 233 | ## Web MQTT configuration: 234 | config: | 235 | # web_mqtt.ssl.port = 12345 236 | # web_mqtt.ssl.backlog = 1024 237 | # web_mqtt.ssl.certfile = /etc/cert/cacert.pem 238 | # web_mqtt.ssl.keyfile = /etc/cert/cert.pem 239 | # web_mqtt.ssl.cacertfile = /etc/cert/key.pem 240 | # web_mqtt.ssl.password = changeme 241 | 242 | ## STOMP Plugin 243 | ## Ref: http://www.rabbitmq.com/stomp.html 244 | rabbitmqSTOMPPlugin: 245 | enabled: false 246 | 247 | ## STOMP configuration: 248 | config: | 249 | # stomp.default_user = guest 250 | # stomp.default_pass = guest 251 | 252 | ## Web STOMP Plugin 253 | ## Ref: http://www.rabbitmq.com/web-stomp.html 254 | rabbitmqWebSTOMPPlugin: 255 | enabled: false 256 | 257 | ## Web STOMP configuration: 258 | config: | 259 | # web_stomp.ws_frame = binary 260 | # web_stomp.cowboy_opts.max_keepalive = 10 261 | 262 | ## AMQPS support 263 | ## Ref: http://www.rabbitmq.com/ssl.html 264 | rabbitmqAmqpsSupport: 265 | enabled: false 266 | 267 | # NodePort 268 | amqpsNodePort: 5671 269 | 270 | # SSL configuration 271 | config: | 272 | listeners.ssl.default = 5671 273 | ssl_options.cacertfile = /etc/cert/cacert.pem 274 | ssl_options.certfile = /etc/cert/cert.pem 275 | ssl_options.keyfile = /etc/cert/key.pem 276 | ssl_options.verify = verify_peer 277 | ssl_options.fail_if_no_peer_cert = true 278 | ## ssl_options.depth = 2 279 | ## Number of replicas 280 | replicaCount: 2 281 | 282 | image: 283 | repository: rabbitmq 284 | tag: 3.7-alpine 285 | pullPolicy: IfNotPresent 286 | 287 | ## This image is used to run a pre-installation script before a rabbit node starts. it always picks up the latest tag of busybox (https://hub.docker.com/_/busybox) 288 | busyboxImage: busybox 289 | 290 | ## Optionally specify an array of imagePullSecrets. 291 | ## Secrets must be manually created in the namespace. 292 | ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ 293 | ## 294 | # pullSecrets: 295 | # - myRegistrKeySecretName 296 | 297 | ## Duration in seconds the pod needs to terminate gracefully 298 | terminationGracePeriodSeconds: 10 299 | 300 | service: 301 | annotations: {} 302 | annotations_lb: 303 | { 304 | } 305 | clusterIP: 306 | 307 | ## List of IP addresses at which the service is available 308 | ## Ref: https://kubernetes.io/docs/user-guide/services/#external-ips 309 | ## 310 | externalIPs: [] 311 | 312 | loadBalancerIP: "" 313 | loadBalancerSourceRanges: [] 314 | type: LoadBalancer 315 | 316 | ## https://www.asykim.com/blog/deep-dive-into-kubernetes-external-traffic-policies 317 | externalTrafficPolicy: Local 318 | 319 | podManagementPolicy: OrderedReady 320 | 321 | ## Statefulsets rolling update update strategy 322 | ## Ref: https://kubernetes.io/docs/tutorials/stateful-application/basic-stateful-set/#rolling-update 323 | ## 324 | updateStrategy: RollingUpdate 325 | 326 | ## Statefulsets Pod Priority 327 | ## Ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#priorityclass 328 | ## priorityClassName: "" 329 | 330 | ## We usually recommend not to specify default resources and to leave this as 331 | ## a conscious choice for the user. This also increases chances charts run on 332 | ## environments with little resources, such as Minikube. If you do want to 333 | ## specify resources, uncomment the following lines, adjust them as necessary, 334 | ## and remove the curly braces after 'resources:'. 335 | ## If you decide to set the memory limit, make sure to also change the 336 | ## rabbitmqMemoryHighWatermark following the formula: 337 | ## rabbitmqMemoryHighWatermark = 0.4 * resources.limits.memory 338 | ## 339 | resources: {} 340 | # limits: 341 | # cpu: 100mm 342 | # memory: 1Gi 343 | # requests: 344 | # cpu: 100mm 345 | # memory: 1Gi 346 | initContainer: 347 | resources: {} 348 | # limits: 349 | # cpu: 100mm 350 | # memory: 128Mi 351 | # requests: 352 | # cpu: 100mm 353 | # memory: 128Mi 354 | 355 | ## Data Persistency 356 | persistentVolume: 357 | enabled: false 358 | ## If defined, storageClassName: 359 | ## If set to "-", storageClassName: "", which disables dynamic provisioning 360 | ## If undefined (the default) or set to null, no storageClassName spec is 361 | ## set, choosing the default provisioner. (gp2 on AWS, standard on 362 | ## GKE, AWS & OpenStack) 363 | ## 364 | # storageClass: "-" 365 | name: data 366 | accessModes: 367 | - ReadWriteOnce 368 | size: 8Gi 369 | annotations: {} 370 | 371 | ## Node labels for pod assignment 372 | ## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector 373 | ## 374 | nodeSelector: {} 375 | 376 | ## Node tolerations for pod assignment 377 | ## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#taints-and-tolerations-beta-feature 378 | ## 379 | tolerations: [] 380 | 381 | ## Extra Annotations to be added to pod 382 | podAnnotations: {} 383 | 384 | ## Pod affinity 385 | ## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity 386 | podAntiAffinity: soft 387 | 388 | ## Create default configMap 389 | ## 390 | existingConfigMap: false 391 | 392 | ## Add additional labels to all resources 393 | ## 394 | extraLabels: {} 395 | 396 | ## Role Based Access 397 | ## Ref: https://kubernetes.io/docs/admin/authorization/rbac/ 398 | ## 399 | rbac: 400 | create: true 401 | 402 | ## Service Account 403 | ## Ref: https://kubernetes.io/docs/admin/service-accounts-admin/ 404 | ## 405 | serviceAccount: 406 | create: true 407 | 408 | ## The name of the ServiceAccount to use. 409 | ## If not set and create is true, a name is generated using the fullname template 410 | # name: 411 | 412 | ingress: 413 | ## Set to true to enable ingress record generation 414 | enabled: false 415 | 416 | path: / 417 | 418 | ## The list of hostnames to be covered with this ingress record. 419 | ## Most likely this will be just one host, but in the event more hosts are needed, this is an array 420 | hostName: rabbitingress.mydomain.com 421 | 422 | ## Set this to true in order to enable TLS on the ingress record 423 | tls: false 424 | 425 | ## If TLS is set to true, you must declare what secret will store the key/certificate for TLS 426 | tlsSecret: myTlsSecret 427 | 428 | ## Ingress annotations done as key:value pairs 429 | annotations: 430 | 431 | livenessProbe: 432 | initialDelaySeconds: 120 433 | periodSeconds: 10 434 | timeoutSeconds: 5 435 | failureThreshold: 6 436 | 437 | readinessProbe: 438 | failureThreshold: 6 439 | initialDelaySeconds: 20 440 | timeoutSeconds: 3 441 | periodSeconds: 5 442 | 443 | # Specifies an existing secret to be used for RMQ password and Erlang Cookie 444 | existingSecret: "" 445 | 446 | prometheus: 447 | ## Configures Prometheus Exporter to expose and scrape stats. 448 | exporter: 449 | enabled: false 450 | env: {} 451 | image: 452 | repository: kbudde/rabbitmq-exporter 453 | tag: v0.28.0 454 | pullPolicy: IfNotPresent 455 | 456 | ## Port Prometheus scrapes for metrics 457 | port: 9419 458 | 459 | ## Export the operator endpoint through external load balancer / nodeport? 460 | exportOperator: false 461 | 462 | ## Comma-separated list of extended scraping capabilities supported by the target RabbitMQ server 463 | capabilities: "bert,no_sort" 464 | 465 | ## Allow overriding of container resources 466 | resources: {} 467 | # limits: 468 | # cpu: 200m 469 | # memory: 1Gi 470 | # requests: 471 | # cpu: 100m 472 | # memory: 100Mi 473 | 474 | ## Prometheus is using Operator. Setting to true will create Operator specific resources like ServiceMonitors and Alerts 475 | operator: 476 | ## Are you using Prometheus Operator? [Blog Post](https://coreos.com/blog/the-prometheus-operator.html) 477 | enabled: false 478 | 479 | ## Configures Alerts, which will be setup via Prometheus Operator / ConfigMaps. 480 | alerts: 481 | ## Prometheus exporter must be enabled as well 482 | enabled: false 483 | 484 | ## Selector must be configured to match Prometheus Install, defaulting to whats done by Prometheus Operator 485 | ## See [CoreOS Prometheus Chart](https://github.com/coreos/prometheus-operator/tree/master/helm) 486 | selector: 487 | role: alert-rules 488 | labels: {} 489 | 490 | serviceMonitor: 491 | ## Interval at which Prometheus scrapes RabbitMQ Exporter 492 | interval: 10s 493 | 494 | ## Defaults to whats used if you follow CoreOS [Prometheus Install Instructions](https://github.com/coreos/prometheus-operator/tree/master/helm#tldr) 495 | ## [Prometheus Selector Label](https://github.com/coreos/prometheus-operator/blob/master/helm/prometheus/templates/prometheus.yaml#L65) 496 | ## [Kube Prometheus Selector Label](https://github.com/coreos/prometheus-operator/blob/master/helm/kube-prometheus/values.yaml#L298) 497 | selector: 498 | prometheus: kube-prometheus 499 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # RabbitMQ-with-istio-MTLS 2 | RabbitMQ stateful deployment with istio service mesh, and with MTLS enabled. 3 | This repository is a clone of https://github.com/helm/charts/tree/master/stable/rabbitmq-ha with an istio MTLS integration. 4 | All charts are under **helm** in this repository, and installation is similar to rabbitmq-ha. 5 | 6 | ## Introduction 7 | Istio (atleast 1.0) does not support fully stateful set deployments - which is discussed in many threads around the web. 8 | There were several workarounds that were suggested, and in this respository i'll explain how i managed to deploy a rabbitMQ deployment with istio and MTLS enabled. 9 | 10 | > note - This chart was not tested with whole combinations, use with cation. 11 | 12 | A typical deployment of this chart can look like this: 13 | ![istio](https://github.com/arielb135/RabbitMQ-with-istio-MTLS/blob/master/sketches/istio.png) 14 | 1. the istio gateway exposes the UI 15 | 2. the "unknown" is another load balancer that exposes only the AMQPS port in case of rabbit implemented MTLS (this should be fixed, not in scope) 16 | 3. the handler is a java app that access rabbit in AMQP port (using istio MTLS) 17 | 18 | > this dashboard came from kiali, which should be enabled when installing istio, more info: https://istio.io/docs/tasks/telemetry/kiali/ 19 | 20 | ## Installing ISTIO 21 | Istio is pretty easy to install, i've used istio 1.0 and followed the following tutorial, while enabling auto sidecar injection: 22 | * https://istio.io/docs/setup/kubernetes/install/helm/ 23 | * https://istio.io/docs/setup/kubernetes/additional-setup/sidecar-injection/#automatic-sidecar-injection 24 | 25 | ## Labeling namespaces for injection 26 | We will want that the istio sidecar will be injected automatically to rabbitMQ namespace (rabbitns let's say) - so we need to label that namespace (after creating it) to allow it: 27 | ```bash 28 | $ kubectl create ns rabbitns 29 | ``` 30 | Then we will label this namespace so all pods will automatically have the istio sidecar container: 31 | ```bash 32 | $ kubectl label namespace rabbitns istio-injection=enabled 33 | ``` 34 | 35 | ## Values.yaml istio support 36 | The part of the yaml that supports istio is this: 37 | ``` yaml 38 | istio: 39 | enabled: true 40 | mtls: true 41 | ingress: 42 | enabled: true 43 | managementHostName: rabbit.mycooldomain.com 44 | ``` 45 | * ingress part defines a record in the ingress gateway 46 | ## Exposing rabbitMQ UI in the ingress gateway 47 | Thankfully, istio gave us an istio gateway that can be used as the entry point if we want to go inside the cluster, we have additional functionality using the gateway, for example - we can terminate TLS there (to the outside world) (not done here). 48 | 49 | ``` yaml 50 | apiVersion: networking.istio.io/v1alpha3 51 | kind: Gateway 52 | metadata: 53 | labels: 54 | app: rabbitmq-ha 55 | chart: rabbitmq-ha-1.12.1 56 | heritage: Tiller 57 | release: rabbitmq 58 | name: rabbitmq-gw 59 | namespace: rabbitns 60 | spec: 61 | selector: 62 | istio: ingressgateway # use default istio gateway 63 | servers: 64 | - hosts: 65 | - rabbit.mycooldomain.com 66 | port: 67 | name: http 68 | number: 80 69 | protocol: HTTP 70 | ``` 71 | * This tells the istio gateway to do something with traffic that comes with the host rabbit.mycooldomain.com on port 80. 72 | 73 | To actually perform the routing to the service, we must use a VirtualService to do so: 74 | ``` yaml 75 | apiVersion: networking.istio.io/v1alpha3 76 | kind: VirtualService 77 | metadata: 78 | labels: 79 | app: rabbitmq-ha 80 | chart: rabbitmq-ha-1.12.1 81 | heritage: Tiller 82 | release: rabbitmq 83 | name: rabbitmq-vs 84 | namespace: rabbitns 85 | spec: 86 | gateways: 87 | - rabbitmq-gw 88 | hosts: 89 | - rabbit.mycooldomain.com 90 | http: 91 | - route: 92 | - destination: 93 | host: rabbitmq-internal.rabbitns.svc.cluster.local 94 | port: 95 | number: 15672 # rabbitmq management UI port 96 | ``` 97 | * In here we're telling to rabbit.mycooldomain.com to go to the internal service rabbitmq-internal.rabbitns.svc.cluster.local (which exposes the rabbit UI) with port 15672 (management UI). 98 | 99 | ![istio](https://github.com/arielb135/RabbitMQ-with-istio-MTLS/blob/master/sketches/ui.png) 100 | 101 | The istio objects that were created come to respond to 3 main issues - 102 | ## Issue 1 - Stateful sets pod discovery 103 | Usually when creating a deployment - we have multiple stateless service that are independent from each other. 104 | Stateful apps such as databases, rabbit are participating in a cluster, which means they require stable DNS of each pod so they can exchange information between them, This is not possible in a regular deployment, as the DNS is not stable (nor the IPs). 105 | this presents us with the "stateful set" option - which assures: 106 | 107 | - Stable DNS names for each pod - for example - rabbitmq-0.rabbitmq-discovery.rabbitns.svc.cluster.local 108 | - rabbitmq-0 - is the name of the statefulset (deployment) with dash and then a running number from 0 to number of replicas - 1 (this assures a known and predictable DNS name) 109 | - arabbitmq-discovery - name of the headless service that gives the stable DNS to the pods 110 | - rabbitns - the namespace the app is deployed into 111 | - Stable storage persistence - which means that when pod is rescheduled, it will reuse the same storage attached to it 112 | - ordered deployment and shutdown (one by one - which is good if a cluster needs to be set up - as the first node creates the cluster, the second joins, etc..) 113 | 114 | More info: https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/ 115 | * Headless service - this is a service without a cluster IP that is responsible in giving the pods their DNS name. 116 | 117 | ### Issue + workaround 118 | Istio discovers well regular services (that attach 1 single DNS to all the pods and then they are accessed in round robin style), but it is not familliar with per pod DNS - thus it rejects it. 119 | We will thus create a service entry per pod, and expose all ports that are needed - telling istio that they are inside the service mesh, example of a ServiceEntry 120 | In rabbitMQ - this looks like this: 121 | 122 | ``` yaml 123 | apiVersion: networking.istio.io/v1alpha3 124 | kind: ServiceEntry 125 | metadata: 126 | labels: 127 | app: rabbitmq-ha 128 | chart: rabbitmq-ha-1.12.1 129 | heritage: Tiller 130 | release: rabbitmq 131 | name: rabbitmq 132 | namespace: rabbitns 133 | spec: 134 | hosts: 135 | - rabbitmq-0.rabbitmq-discovery.rabbitns.svc.cluster.local 136 | - rabbitmq-1.rabbitmq-discovery.rabbitns.svc.cluster.local 137 | location: MESH_INTERNAL 138 | ports: 139 | - name: http 140 | number: 15672 141 | protocol: TCP 142 | - name: amqp 143 | number: 5672 144 | protocol: TCP 145 | - name: epmd 146 | number: 4369 147 | protocol: TCP 148 | - name: amqps 149 | number: 5671 150 | protocol: TCP 151 | - name: exporter 152 | number: 9419 153 | protocol: TCP 154 | - name: inter-node 155 | number: 25672 156 | protocol: TCP 157 | resolution: NONE 158 | ``` 159 | * the full DNS of each pod (received from a kubernetes [headless service](https://kubernetes.io/docs/concepts/services-networking/service/#headless-services)). 160 | * All ports that the service used need to be declared, so istio can register them. 161 | * The helm chart creates the list of hosts dynamically by replica count. 162 | * Not tested, but *.rabbitmq-discovery.rabbitns.svc.cluster.local might cover all pods (instead of creating it 1 by 1) 163 | 164 | ### more info: 165 | * general istio thread for stateful sets - https://github.com/istio/istio/issues/10659 166 | * Headless service solution - https://github.com/istio/istio/issues/7495 167 | 168 | ## Issue 2 - Mutual TLS for POD IP communication 169 | ### Issue 170 | 171 | When enabling mutual TLS, we need to configure the clients - so they will also talk in MTLS to the service that was enabled with MTLS. 172 | This is usually done by DestinationRule. Some applications require to talk to the localhost for some logic they have, istio lets all loopback interfaces pass without MTLS requirement (127.0.0.1 and localhost). 173 | 174 | when a pod uses its pod IP for local communication, it will fail as the MTLS policy is default deny - istio yet to support talking locally with POD IP and mtls, Cassandra is a simple example to that: 175 | https://aspenmesh.io/2019/03/running-stateless-apps-with-service-mesh-kubernetes-cassandra-with-istio-mtls-enabled/ 176 | 177 | ### Workaround 178 | 179 | To workaround this, we have 2 options: 180 | 181 | - change the application to use localhost/127.0.0.1 instead of pod ip - this is possible for example in cassandra, but not in RabbitMQ 182 | - if #1 is not possible, we will need to exclude MTLS communication from this specific service (usually a port), in rabbitMQ it's epmd which is the service that does cluster discovery. This service talks in local pod IP - which causes failure in the MTLS, this issue talks about it: https://github.com/istio/istio/issues/6828 183 | 184 | so in rabbitMQ you have to exclude this port (thus, service discovery will not be done encrypted) - always consult with security architect / anyone else to see what it means to the deployment. 185 | 186 | **Policy:** 187 | This is done aswell by the policy object, we can see an example 188 | ``` yaml 189 | apiVersion: authentication.istio.io/v1alpha1 190 | kind: Policy 191 | metadata: 192 | labels: 193 | app: rabbitmq-ha 194 | chart: rabbitmq-ha-1.12.1 195 | heritage: Tiller 196 | release: rabbitmq 197 | name: default 198 | namespace: rabbitns 199 | spec: 200 | peers: 201 | - mtls: {} 202 | ``` 203 | * Which enables the mtls to the *whole* rabbitns namespace 204 | 205 | We know that we have a problem with the epmd (4369 port) service which tries to access the local pod IP, instead of localhost - so we'll exclude it. 206 | We'll also exclude the secured AMQPS port - as if it's configured - we will now do a TLS over TLS (not needed). 207 | ``` yaml 208 | apiVersion: authentication.istio.io/v1alpha1 209 | kind: Policy 210 | metadata: 211 | labels: 212 | app: rabbitmq-ha 213 | chart: rabbitmq-ha-1.12.1 214 | heritage: Tiller 215 | release: rabbitmq 216 | name: rabbitmq-disable-mtls 217 | namespace: rabbitns 218 | spec: 219 | targets: 220 | - name: rabbitmq-discovery 221 | ports: 222 | - number: 4369 223 | - number: 5671 224 | - name: rabbitmq 225 | ports: 226 | - number: 5671 227 | - name: rabbitmq-internal 228 | ports: 229 | - number: 5671 230 | ``` 231 | * In here, we are addressing 3 services - the first is the headless service, which we want to exclude the epmd port and also the amqps (probably unecessary). The second is the regular service, that in this chart we expose it via load balancer for example, so we want to omit MTLS from it, third one is the internal service - which we also want to exclude 5671 mtls port. 232 | * Now we have the entire rabbitMQ services require MTLS to work - excluding 4369 and 5671 233 | 234 | > Important - there can be only one per namespace policy - we are utilizing it in this chart, meaning - you should deploy rabbit in its own namespace to avoid problems - https://istio.io/docs/concepts/security/#target-selectors 235 | 236 | **Destination Rule:** 237 | To configure the "client" (the rabbitMQ pods themselves so they can initiate MTLS connection as a "client") - we need to create several destination rules - this is a bit tricky. 238 | The deployment is divided to 4 destination rules (the "clients"), to avoid conflicts in cluster (even if some of the ports are not used in some services) 239 | * Per pod - ***.rabbitmq-discovery.rabbitns.svc.cluster.local** - which sets a full MTLS for all ports (for the client side) 240 | * exposed through ingress service - **rabbitmq.rabbitns.svc.cluster.local** - full MTLS except amqps port (5671) 241 | * headless service - **rabbitmq-discovery.rabbitns.svc.cluster.local** - full MTLS except amqps port (5671) and epmd port (4369) 242 | * internal service - **rabbitmq-internal.rabbitns.svc.cluster.local** - full MTLS except amqps port (5671) 243 | ``` yaml 244 | apiVersion: v1 245 | items: 246 | apiVersion: networking.istio.io/v1alpha3 247 | kind: DestinationRule 248 | metadata: 249 | labels: 250 | app: rabbitmq-ha 251 | chart: rabbitmq-ha-1.12.1 252 | heritage: Tiller 253 | release: rabbitmq 254 | name: rabbitmq-mtls-per-pod 255 | namespace: rabbitns 256 | spec: 257 | host: '*.rabbitmq-discovery.rabbitns.svc.cluster.local' 258 | trafficPolicy: 259 | tls: 260 | mode: ISTIO_MUTUAL 261 | --- 262 | apiVersion: v1 263 | items: 264 | apiVersion: networking.istio.io/v1alpha3 265 | kind: DestinationRule 266 | metadata: 267 | labels: 268 | app: rabbitmq-ha 269 | chart: rabbitmq-ha-1.12.1 270 | heritage: Tiller 271 | release: rabbitmq 272 | name: rabbitmq-mtls 273 | namespace: rabbitns 274 | spec: 275 | host: 'rabbitmq.rabbitns.svc.cluster.local' 276 | trafficPolicy: 277 | portLevelSettings: 278 | - port: 279 | number: 5671 280 | tls: 281 | mode: DISABLE 282 | tls: 283 | mode: ISTIO_MUTUAL 284 | --- 285 | apiVersion: v1 286 | items: 287 | apiVersion: networking.istio.io/v1alpha3 288 | kind: DestinationRule 289 | metadata: 290 | labels: 291 | app: rabbitmq-ha 292 | chart: rabbitmq-ha-1.12.1 293 | heritage: Tiller 294 | release: rabbitmq 295 | name: rabbitmq-mtls 296 | namespace: rabbitns 297 | spec: 298 | host: 'rabbitmq-discovery.rabbitns.svc.cluster.local' 299 | trafficPolicy: 300 | portLevelSettings: 301 | - port: 302 | number: 4369 303 | tls: 304 | mode: DISABLE 305 | - port: 306 | number: 5671 307 | tls: 308 | mode: DISABLE 309 | tls: 310 | mode: ISTIO_MUTUAL 311 | --- 312 | apiVersion: v1 313 | items: 314 | apiVersion: networking.istio.io/v1alpha3 315 | kind: DestinationRule 316 | metadata: 317 | labels: 318 | app: rabbitmq-ha 319 | chart: rabbitmq-ha-1.12.1 320 | heritage: Tiller 321 | release: rabbitmq 322 | name: rabbitmq-mtls-internal 323 | namespace: rabbitns 324 | spec: 325 | host: 'rabbitmq-internal.rabbitns.svc.cluster.local' 326 | trafficPolicy: 327 | portLevelSettings: 328 | - port: 329 | number: 5671 330 | tls: 331 | mode: DISABLE 332 | tls: 333 | mode: ISTIO_MUTUAL 334 | ``` 335 | * The above means that we would like a mutual TLS for everything that is directed to either the pod themselves, or 336 | * We want to avoid conflicts, so we will disable ports that don't require MTLS for each service / host. 337 | ** 5671 is the AMQPS port - which we want the client to use it regulary. 338 | * Because we deployed it in the rabbitns namespace, it will apply to whole "clients" of that namespace. 339 | 340 | > Note - it's ok to create one destination rule that catchs all ***.rabbitns.svc.cluster.local**, but you will have conflicts when checking tls settings - this might be ok even in istio strict TLS mode - but requires more testing 341 | 342 | If we'll have another rabbit client in another namespace (let's say javarabbitclient), we would create the a simpler rule for the internal service , but in that specific namespace: 343 | ``` yaml 344 | apiVersion: v1 345 | items: 346 | apiVersion: networking.istio.io/v1alpha3 347 | kind: DestinationRule 348 | metadata: 349 | labels: 350 | app: rabbitmq-ha 351 | chart: rabbitmq-ha-1.12.1 352 | heritage: Tiller 353 | release: rabbitmq 354 | name: java-dr 355 | namespace: javarabbitclient 356 | spec: 357 | host: 'rabbitmq-internal.rabbitns.svc.cluster.local' 358 | trafficPolicy: 359 | portLevelSettings: 360 | - port: 361 | number: 5671 362 | tls: 363 | mode: DISABLE 364 | tls: 365 | mode: ISTIO_MUTUAL 366 | ``` 367 | 368 | This sketch illustrates what happens: 369 | ![TLS](https://github.com/arielb135/RabbitMQ-with-istio-MTLS/blob/master/sketches/tls.png) 370 | * The destination rule is deployed in the client's namespace, and directs that it wants to talk as MTLS to the service rabbitmq-internal.rabbitns.svc.cluster.local 371 | * The policy says to enable MTLS for the whole namespace, but exclude certain ports 372 | 373 | ## Issue 3 - Headless service DNS entry cannot participate in MTLS 374 | 375 | ### Issue 376 | Headless service is the service that gives the pods their stable DNS record. 377 | 378 | as kubernetes gives the headless service also a full DNS name (like regular service), which can be used as a round robin style aswell. - it's not a good idea to use that DNS to talk inside the cluster anyway, as it used for distribution of DNS entries to per-pod, and for good order - it's recommended to create a seperate internal service for internal communication. 379 | 380 | ### Workaround 381 | Create another service without a type (clusterIP), which will not attach it to any load balancer - and you'll have a internal service that we can use (with the DNS - rabbitmq-internal.rabbitns.svc.cluster.local) 382 | istio will discover it well. 383 | 384 | ``` yaml 385 | apiVersion: v1 386 | kind: Service 387 | metadata: 388 | labels: 389 | app: rabbitmq-ha 390 | chart: rabbitmq-ha-1.12.1 391 | heritage: Tiller 392 | release: rabbitmq 393 | name: rabbitmq-internal 394 | namespace: rabbitns 395 | spec: 396 | ports: 397 | - name: http 398 | port: 15672 399 | protocol: TCP 400 | targetPort: http 401 | - name: amqp 402 | port: 5672 403 | protocol: TCP 404 | targetPort: amqp 405 | - name: amqps 406 | port: 5671 407 | protocol: TCP 408 | targetPort: amqps 409 | selector: 410 | app: rabbitmq-ha 411 | release: rabbitmq 412 | ``` 413 | 414 | ## Verifying TLS 415 | taken from here: https://istio.io/docs/tasks/security/mutual-tls/#verify-mutual-tls-configuration 416 | 417 | we will use the istioctl tool, with the tls-check option, 418 | ``` bash 419 | $ istioctl authn tls-check handler-6f936c59f5-4d88d.javarabbitclient 420 | ``` 421 | * This checks the tls of the handler pod (our java service) dot namespace ({pod}.{namespace}) in the mesh. as we're using the rabbitmq-internal - we can look at the entries of it: 422 | 423 | we can now see the output: 424 | ``` bash 425 | HOST:PORT STATUS SERVER CLIENT AUTHN POLICY DESTINATION RULE 426 | rabbitmq-internal.rabbitns.svc.cluster.local:5671 OK HTTP HTTP rabbitmq-disable-mtls/rabbitns handler/javarabbitclient 427 | rabbitmq-internal.rabbitns.svc.cluster.local:5672 OK mTLS mTLS default/rabbitns handler/javarabbitclient 428 | rabbitmq-internal.rabbitns.svc.cluster.local:9419 OK mTLS mTLS default/rabbitns handler/javarabbitclient 429 | rabbitmq-internal.rabbitns.svc.cluster.local:15672 OK mTLS mTLS default/rabbitns handler/javarabbitclient 430 | ``` 431 | We can see several interesting entries here: 432 | * Port 5671 - which is the secured AMQPS port that we decided to exclude from MTLS communication 433 | * Port 5672 - regular unencrypted AMQP port - will now be encrypted under istio mTLS 434 | * Port 9419 - is for metrics scraping (Note - this was not tested if it works, so prometheus integration and istio configuration is out of scope for this charts) 435 | * Port 15672 - the management UI 436 | 437 | ## Unrelated chart features 438 | ### Reoccuring definitions 439 | There's an ability to add reoccuring entries to definitions.json with "definitions_reoccuring": 440 | ``` yaml 441 | definitions_reoccuring: 442 | enabled: true 443 | replaceString: "#" 444 | startFrom: 101 445 | until: 120 446 | ## Must be string values ( e.g. ["102", "103"] 447 | skipIndexes: ["110"] 448 | users: |- 449 | { 450 | "name": "user#", 451 | "tags": "" 452 | } 453 | permissions: |- 454 | { 455 | "user": "user#", 456 | "vhost": "/", 457 | "configure": ".*", 458 | "write": ".*", 459 | "read": ".*" 460 | } 461 | ``` 462 | In the above we will create 19 users (from 101 to 120, excluding 110) that are called user user101, user102 etc... 463 | There's the exclusion field to skip some users. 464 | This supports any entry in definitions.json - and the string to replace with number is # 465 | --------------------------------------------------------------------------------