├── .env ├── .gitignore ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── Makefile ├── README.md ├── playlist.md └── s01 ├── e01 ├── .env ├── Makefile ├── README.md ├── tgir-s01e01-logs-rmq-lifecycle.png ├── tgir-s01e01-logs.png ├── tgir-s01e01-rmq-management-37x.png ├── tgir-s01e01-rmq-management-38x.png └── video.png ├── e02 ├── .env ├── Erlang-Memory-Allocators-100k-messages-lazy-q.png ├── Erlang-Memory-Allocators-50mil-messages-durable-q.png ├── Erlang-Memory-Allocators-50mil-messages-lazy-q.png ├── Makefile ├── README.md ├── RabbitMQ-Overview-100k-messages-lazy-q.png ├── RabbitMQ-Overview-50mil-messages-durable-q.png ├── RabbitMQ-Overview-50mil-messages-lazy-q.png └── video.png ├── e03 ├── .env ├── .gitignore ├── Makefile ├── README.md └── video.jpg ├── e04 ├── .env ├── Makefile ├── README.md └── video.jpg ├── e05 ├── .env ├── .gitignore ├── Makefile ├── README.md ├── k8s │ ├── benchmark.job │ ├── persistentvolumeclaim.yml │ ├── service.yml │ ├── statefulset.volume │ └── statefulset.yml └── video.jpg ├── e06 ├── .env ├── .gitignore ├── Makefile ├── README.md ├── k8s │ ├── configmap.cluster-formation.yml │ ├── configmap.cluster-name.yml │ ├── configmap.disk-free-limit.yml │ ├── configmap.enabled-plugins.yml │ ├── configmap.log.yml │ ├── configmap.queue-master-locator.yml │ ├── configmap.total-memory-available-override-value.yml │ ├── configmap.vm-memory-high-watermark-paging-ratio.yml │ ├── configmap.vm-memory-high-watermark.yml │ ├── deployment.consumer │ ├── deployment.publisher │ ├── role.yml │ ├── rolebinding.yml │ ├── service.reliable-rabbit-public.yml │ ├── service.reliable-rabbit.yml │ ├── serviceaccount.yml │ ├── statefulset.rabbitmq-upgrade │ ├── statefulset.yml │ └── storageclass.ssd.yml └── video.jpg ├── e07 ├── .env ├── .gitignore ├── Makefile ├── README.md ├── k8s │ ├── prometheus-stack │ │ ├── erlang-distribution-grafana-dashboard.yml │ │ ├── erlang-memory-allocators-grafana-dashboard.yml │ │ ├── rabbitmq-overview-grafana-dashboard.yml │ │ ├── rabbitmq-perftest-grafana-dashboard.yml │ │ ├── rabbitmq-quorum-queues-raft-grafana-dashboard.yml │ │ ├── traefik-grafana-dashboard.yml │ │ ├── traefik-servicemonitor.yml │ │ └── values.yml │ └── rabbitmq │ │ ├── default-metrics.yml │ │ ├── minimal-metrics.yml │ │ ├── perftest.help │ │ ├── prometheus-metrics.yml │ │ ├── quorum.yml │ │ ├── stream-perftest.help │ │ └── stream.yml └── video.jpg ├── e08 ├── .env ├── .gitignore ├── Makefile ├── README.md ├── video.jpg └── yaml │ ├── cert-manager-cloudflare-issuer.yaml │ ├── rabbitmq-bugs-cert.yaml │ ├── rabbitmq-bugs-insecure.yaml │ ├── rabbitmq-bugs-secure-perftest.yaml │ ├── rabbitmq-bugs-secure.yaml │ ├── rabbitmq-bugs-traefik-ingress-amqps.yaml │ ├── rabbitmq-bugs-traefik-ingress-https.yaml │ ├── rabbitmq-bunny-cert.yaml │ ├── rabbitmq-bunny-insecure.yaml │ ├── rabbitmq-bunny-secure-perftest.yaml │ ├── rabbitmq-bunny-secure.yaml │ ├── rabbitmq-bunny-traefik-ingress-amqps.yaml │ ├── rabbitmq-bunny-traefik-ingress-https.yaml │ ├── steps │ └── values.yaml └── e09 ├── .env ├── .gitignore ├── Makefile ├── README.md ├── k8s ├── chaos-experiments │ ├── cluster-latency.yaml │ ├── cluster-partition.yaml │ ├── cpu-stealing.yaml │ ├── disk-latency.yaml │ ├── intra-node-partition.yaml │ ├── kill-rmq-pod-every-5m.yaml │ └── memory-filling.yaml ├── monitoring-stack │ ├── dashboards │ │ ├── erlang-distribution-grafana-dashboard.yml │ │ ├── erlang-memory-allocators-grafana-dashboard.yml │ │ ├── rabbitmq-overview-grafana-dashboard.yml │ │ ├── rabbitmq-perftest-grafana-dashboard.yml │ │ ├── rabbitmq-quorum-queues-raft-grafana-dashboard.yml │ │ └── traefik-grafana-dashboard.yml │ ├── traefik-servicemonitor.yml │ └── values.yml └── rabbitmq │ ├── clients │ ├── perf-test-classic-queues-second-client.yaml │ ├── perf-test-classic-queues.yaml │ └── perf-test-quorum.yaml │ ├── clusters │ ├── partition-autoheal.yaml │ ├── partition-ignore.yaml │ └── partition-pause-minority.yaml │ ├── pod-disruption-budget.yaml │ └── ssd-gke.yaml ├── rabbitmq-chaos-mesh-community-meetup-january-2021.jpg └── video.jpg /.env: -------------------------------------------------------------------------------- 1 | # shellcheck shell=sh 2 | 3 | main() { 4 | if shell_is_bash 5 | then 6 | load_bash_autocomplete 7 | fi 8 | load_make_env 9 | } 10 | 11 | shell_is_bash() { 12 | # we don't want to depend on pgrep, it doesn't ship on macOS out of the box as far as I can remember 13 | # shellcheck disable=SC2009 14 | ps -p $$ -o comm= | grep -q "bash" 15 | } 16 | 17 | load_bash_autocomplete() { 18 | eval "$(make bash-autocomplete)" 19 | } 20 | 21 | load_make_env() { 22 | eval "$(make env)" 23 | } 24 | 25 | main 26 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | bin 2 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Code of Conduct 2 | 3 | As contributors and maintainers of this project, and in the interest of fostering an open 4 | and welcoming community, we pledge to respect all people who contribute through reporting 5 | issues, posting feature requests, updating documentation, submitting pull requests or 6 | patches, and other activities. 7 | 8 | We are committed to making participation in this project a harassment-free experience for 9 | everyone, regardless of level of experience, gender, gender identity and expression, 10 | sexual orientation, disability, personal appearance, body size, race, ethnicity, age, 11 | religion, or nationality. 12 | 13 | Examples of unacceptable behavior by participants include: 14 | 15 | * The use of sexualized language or imagery 16 | * Personal attacks 17 | * Trolling or insulting/derogatory comments 18 | * Public or private harassment 19 | * Publishing other's private information, such as physical or electronic addresses, 20 | without explicit permission 21 | * Other unethical or unprofessional conduct 22 | 23 | Project maintainers have the right and responsibility to remove, edit, or reject comments, 24 | commits, code, wiki edits, issues, and other contributions that are not aligned to this 25 | Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors 26 | that they deem inappropriate, threatening, offensive, or harmful. 27 | 28 | By adopting this Code of Conduct, project maintainers commit themselves to fairly and 29 | consistently applying these principles to every aspect of managing this project. Project 30 | maintainers who do not follow or enforce the Code of Conduct may be permanently removed 31 | from the project team. 32 | 33 | This Code of Conduct applies both within project spaces and in public spaces when an 34 | individual is representing the project or its community. 35 | 36 | Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by 37 | contacting a project maintainer at [info@rabbitmq.com](mailto:info@rabbitmq.com). All complaints will 38 | be reviewed and investigated and will result in a response that is deemed necessary and 39 | appropriate to the circumstances. Maintainers are obligated to maintain confidentiality 40 | with regard to the reporter of an incident. 41 | 42 | This Code of Conduct is adapted from the 43 | [Contributor Covenant](https://contributor-covenant.org), version 1.3.0, available at 44 | [contributor-covenant.org/version/1/3/0/](https://contributor-covenant.org/version/1/3/0/) 45 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | ## DCO Sign off 3 | 4 | All authors to the project retain copyright to their work. However, to ensure 5 | that they are only submitting work that they have rights to, we are requiring 6 | everyone to acknowledge this by signing their work. 7 | 8 | Any copyright notices in this repo should specify the authors as "RabbitMQ". 9 | 10 | To sign your work, just add a line like this at the end of your commit message: 11 | 12 | ``` 13 | Signed-off-by: Gerhard Lazu 14 | ``` 15 | 16 | This can easily be done with the `--signoff` option to `git commit`. 17 | 18 | By doing this you state that you can certify the following (from https://developercertificate.org/): 19 | 20 | ``` 21 | Developer Certificate of Origin 22 | Version 1.1 23 | 24 | Copyright (C) 2004, 2006 The Linux Foundation and its contributors. 25 | 1 Letterman Drive 26 | Suite D4700 27 | San Francisco, CA, 94129 28 | 29 | Everyone is permitted to copy and distribute verbatim copies of this 30 | license document, but changing it is not allowed. 31 | 32 | 33 | Developer's Certificate of Origin 1.1 34 | 35 | By making a contribution to this project, I certify that: 36 | 37 | (a) The contribution was created in whole or in part by me and I 38 | have the right to submit it under the open source license 39 | indicated in the file; or 40 | 41 | (b) The contribution is based upon previous work that, to the best 42 | of my knowledge, is covered under an appropriate open source 43 | license and I have the right under that license to submit that 44 | work with modifications, whether created in whole or in part 45 | by me, under the same open source license (unless I am 46 | permitted to submit under a different license), as indicated 47 | in the file; or 48 | 49 | (c) The contribution was provided directly to me by some other 50 | person who certified (a), (b) or (c) and I have not modified 51 | it. 52 | 53 | (d) I understand and agree that this project and the contribution 54 | are public and that a record of the contribution (including all 55 | personal information I submit with it, including my sign-off) is 56 | maintained indefinitely and may be redistributed consistent with 57 | this project or the open source license(s) involved. 58 | ``` 59 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | SHELL := bash# we want bash behaviour in all shell invocations 2 | PLATFORM := $(shell uname) 3 | platform = $(shell echo $(PLATFORM) | tr A-Z a-z) 4 | # 5 | # https://stackoverflow.com/questions/4842424/list-of-ansi-color-escape-sequences 6 | RED := \033[1;31m 7 | GREEN := \033[1;32m 8 | YELLOW := \033[1;33m 9 | BOLD := \033[1m 10 | NORMAL := \033[0m 11 | 12 | XDG_CONFIG_HOME := $(CURDIR)/.config 13 | export XDG_CONFIG_HOME 14 | 15 | THISFILE := $(word $(words $(MAKEFILE_LIST)),$(MAKEFILE_LIST)) 16 | THISDIR := $(dir $(THISFILE)) 17 | BASEDIR := $(abspath $(THISDIR)) 18 | LOCAL_BIN := $(BASEDIR)/bin 19 | $(LOCAL_BIN): 20 | mkdir -p $@ 21 | 22 | 23 | 24 | ### DEPS ### 25 | # 26 | ifeq ($(PLATFORM),Darwin) 27 | DOCKER ?= /usr/local/bin/docker 28 | COMPOSE ?= $(DOCKER)-compose 29 | $(DOCKER) $(COMPOSE): 30 | brew cask install docker 31 | else 32 | DOCKER ?= /usr/bin/docker 33 | $(DOCKER): 34 | $(error Please install docker: https://docs.docker.com/install/linux/docker-ce/ubuntu/) 35 | COMPOSE ?= $(DOCKER)-compose 36 | $(COMPOSE): 37 | $(error Please install docker-compose: https://docs.docker.com/compose/install/) 38 | endif 39 | 40 | ifeq ($(PLATFORM),Darwin) 41 | FFMPEG := /usr/local/bin/ffmpeg 42 | $(FFMPEG): 43 | brew install ffmpeg 44 | else 45 | FFMPEG ?= /usr/bin/ffmpeg 46 | $(FFMPEG): 47 | $(error Please install ffmpeg) 48 | endif 49 | 50 | CURL ?= /usr/bin/curl 51 | ifneq ($(PLATFORM),Darwin) 52 | $(CURL): 53 | $(error Please install curl) 54 | endif 55 | 56 | ifeq ($(PLATFORM),Darwin) 57 | OPEN := open 58 | else 59 | OPEN := xdg-open 60 | endif 61 | 62 | ifeq ($(PLATFORM),Darwin) 63 | GCLOUD := /usr/local/bin/gcloud 64 | $(GCLOUD): 65 | brew cask install google-cloud-sdk 66 | else 67 | GCLOUD ?= /usr/bin/gcloud 68 | $(GCLOUD): 69 | $(error Please install gcloud: https://cloud.google.com/sdk/docs/downloads-apt-get) 70 | endif 71 | 72 | ifeq ($(PLATFORM),Darwin) 73 | BAT ?= /usr/local/bin/bat 74 | $(BAT): 75 | brew install bat 76 | else 77 | BAT ?= /usr/bin/bat 78 | $(BAT): 79 | $(error Please install bat: https://github.com/sharkdp/bat) 80 | endif 81 | .PHONY: bat 82 | bat: $(BAT) 83 | 84 | K9S_RELEASES := https://github.com/derailed/k9s/releases 85 | K9S_VERSION := 0.24.2 86 | K9S_BIN_DIR := k9s-$(K9S_VERSION)-$(platform)-x86_64 87 | K9S_URL := $(K9S_RELEASES)/download/v$(K9S_VERSION)/k9s_$(PLATFORM)_x86_64.tar.gz 88 | K9S := $(LOCAL_BIN)/$(K9S_BIN_DIR)/k9s 89 | $(K9S): | $(CURL) $(LOCAL_BIN) 90 | $(CURL) --progress-bar --fail --location --output $(LOCAL_BIN)/$(K9S_BIN_DIR).tar.gz "$(K9S_URL)" 91 | mkdir -p $(LOCAL_BIN)/$(K9S_BIN_DIR) && tar zxf $(LOCAL_BIN)/$(K9S_BIN_DIR).tar.gz -C $(LOCAL_BIN)/$(K9S_BIN_DIR) 92 | touch $(K9S) 93 | chmod +x $(K9S) 94 | $(K9S) version \ 95 | | grep $(K9S_VERSION) 96 | ln -sf $(K9S) $(LOCAL_BIN)/k9s 97 | .PHONY: releases-k9s 98 | releases-k9s: 99 | $(OPEN) $(K9S_RELEASES) 100 | 101 | KUBECTL_RELEASES := https://github.com/kubernetes/kubernetes/releases 102 | # K8S v1.18 is considered stable in October 2020, using latest version available 103 | KUBECTL_VERSION ?= 1.18.10 104 | KUBECTL_BIN = kubectl-$(KUBECTL_VERSION)-$(platform)-amd64 105 | KUBECTL_URL = https://storage.googleapis.com/kubernetes-release/release/v$(KUBECTL_VERSION)/bin/$(platform)/amd64/kubectl 106 | KUBECTL = $(LOCAL_BIN)/$(KUBECTL_BIN) 107 | $(KUBECTL): | $(CURL) $(LOCAL_BIN) 108 | $(CURL) --progress-bar --fail --location --output $(KUBECTL) "$(KUBECTL_URL)" 109 | touch $(KUBECTL) 110 | chmod +x $(KUBECTL) 111 | $(KUBECTL) version | grep $(KUBECTL_VERSION) 112 | ln -sf $(KUBECTL) $(LOCAL_BIN)/kubectl 113 | .PHONY: kubectl 114 | kubectl: $(KUBECTL) 115 | 116 | JQ_RELEASES := https://github.com/stedolan/jq/releases 117 | JQ_VERSION := 1.6 118 | JQ_BIN := jq-$(JQ_VERSION)-$(platform)-x86_64 119 | JQ_URL := $(JQ_RELEASES)/download/jq-$(JQ_VERSION)/jq-$(platform)64 120 | ifeq ($(platform),darwin) 121 | JQ_URL := $(JQ_RELEASES)/download/jq-$(JQ_VERSION)/jq-osx-amd64 122 | endif 123 | JQ := $(LOCAL_BIN)/$(JQ_BIN) 124 | $(JQ): | $(CURL) $(LOCAL_BIN) 125 | $(CURL) --progress-bar --fail --location --output $(JQ) "$(JQ_URL)" \ 126 | && touch $(JQ) \ 127 | && chmod +x $(JQ) \ 128 | && $(JQ) --version \ 129 | | grep $(JQ_VERSION) \ 130 | && ln -sf $(JQ) $(LOCAL_BIN)/jq 131 | .PHONY: jq 132 | jq: $(JQ) 133 | .PHONY: releases-jq 134 | releases-jq: 135 | $(OPEN) $(JQ_RELEASES) 136 | 137 | YQ_RELEASES := https://github.com/mikefarah/yq/releases 138 | YQ_VERSION := 3.4.1 139 | YQ_BIN := yq-$(YQ_VERSION)-$(platform)-amd64 140 | YQ_URL := $(YQ_RELEASES)/download/$(YQ_VERSION)/yq_$(platform)_amd64 141 | YQ := $(LOCAL_BIN)/$(YQ_BIN) 142 | $(YQ): | $(CURL) $(LOCAL_BIN) 143 | $(CURL) --progress-bar --fail --location --output $(YQ) "$(YQ_URL)" 144 | touch $(YQ) 145 | chmod +x $(YQ) 146 | $(YQ) --version | grep $(YQ_VERSION) 147 | ln -sf $(YQ) $(LOCAL_BIN)/yq 148 | .PHONY: yq 149 | yq: $(YQ) 150 | .PHONY: releases-yq 151 | releases-yq: 152 | $(OPEN) $(YQ_RELEASES) 153 | 154 | HELM_RELEASES := https://github.com/helm/helm/releases 155 | HELM_VERSION := 3.4.2 156 | HELM_BIN_DIR := helm-v$(HELM_VERSION)-$(platform)-amd64 157 | HELM_URL := https://get.helm.sh/$(HELM_BIN_DIR).tar.gz 158 | HELM := $(LOCAL_BIN)/$(HELM_BIN_DIR)/$(platform)-amd64/helm 159 | $(HELM): | $(CURL) $(LOCAL_BIN) 160 | $(CURL) --progress-bar --fail --location --output $(LOCAL_BIN)/$(HELM_BIN_DIR).tar.gz "$(HELM_URL)" 161 | mkdir -p $(LOCAL_BIN)/$(HELM_BIN_DIR) && tar zxf $(LOCAL_BIN)/$(HELM_BIN_DIR).tar.gz -C $(LOCAL_BIN)/$(HELM_BIN_DIR) 162 | touch $(HELM) 163 | chmod +x $(HELM) 164 | $(HELM) version | grep $(HELM_VERSION) 165 | ln -sf $(HELM) $(LOCAL_BIN)/helm 166 | .PHONY: helm 167 | helm: $(HELM) 168 | .PHONY: releases-helm 169 | releases-helm: 170 | $(OPEN) $(HELM_RELEASES) 171 | 172 | 173 | 174 | ### TARGETS ### 175 | # 176 | 177 | .DEFAULT_GOAL = help 178 | 179 | .PHONY: help 180 | help: 181 | @awk -F':+ |##' '/^[^\.][0-9a-zA-Z\._\-%]+:+.+##.+$$/ { printf "\033[36m%-32s\033[0m %s\n", $$1, $$3 }' $(MAKEFILE_LIST) \ 182 | | sort 183 | 184 | define MAKE_TARGETS 185 | awk -F':+' '/^[^\.%\t\_][0-9a-zA-Z\._\-\%]*:+.*$$/ { printf "%s\n", $$1 }' $(MAKEFILE_LIST) 186 | endef 187 | define BASH_AUTOCOMPLETE 188 | complete -W \"$$($(MAKE_TARGETS) | sort | uniq)\" make gmake m 189 | endef 190 | .PHONY: bash-autocomplete 191 | bash-autocomplete: 192 | @echo "$(BASH_AUTOCOMPLETE)" 193 | .PHONY: bac 194 | bac: bash-autocomplete 195 | 196 | ifneq ($(GITHUB_USER),) 197 | GRIP_USER := --user $(GITHUB_USER) 198 | endif 199 | ifneq ($(GITHUB_PERSONAL_ACCESS_TOKEN),) 200 | GRIP_PASS := --pass $(GITHUB_PERSONAL_ACCESS_TOKEN) 201 | endif 202 | .PHONY: readme 203 | readme: $(DOCKER) 204 | $(DOCKER) run --interactive --tty --rm \ 205 | --volume $(CURDIR):/data \ 206 | --volume $(HOME)/.grip:/.grip \ 207 | --expose 5000 --publish 5000:5000 \ 208 | --name readme \ 209 | mbentley/grip --context=. 0.0.0.0:5000 $(GRIP_USER) $(GRIP_PASS) 210 | 211 | # https://www.bugcodemaster.com/article/convert-video-animated-gif-using-ffmpeg 212 | # https://trac.ffmpeg.org/wiki/Scaling 213 | .PHONY: gif 214 | gif: $(FFMPEG) 215 | ifndef MP4 216 | $(error MP4 variable must reference a valid mov file path) 217 | endif 218 | time $(FFMPEG) -i $(MP4) \ 219 | -hide_banner \ 220 | -vf "fps=1" \ 221 | $(subst .mp4,-1fps.gif,$(MP4)) 222 | 223 | .env: 224 | ln -sf ../../.env .env 225 | 226 | .PHONY: env 227 | env:: 228 | @true 229 | 230 | FPS ?= 30 231 | FFMPEG_VF = -vf "fps=$(FPS) 232 | MP4_APPEND = -$(FPS)fps 233 | # Scale for videos, use in SCALE= 234 | 1080p = scale='min(1920,iw)':'min(1080,ih)' 235 | 720p = scale='min(1280,iw)':'min(720,ih)' 236 | ifneq ($($(SCALE)),) 237 | FFMPEG_VF := $(FFMPEG_VF),$($(SCALE)) 238 | MP4_APPEND := -$(SCALE)$(MP4_APPEND) 239 | endif 240 | FFMPEG_VF := $(FFMPEG_VF)" 241 | 242 | .PHONY: mp4 243 | mp4: $(FFMPEG) 244 | ifndef MOV 245 | $(error MOV variable must reference a valid mov file path) 246 | endif 247 | time $(FFMPEG) -i $(MOV) \ 248 | -vcodec h264 $(FFMPEG_VF) \ 249 | $(subst .mov,$(MP4_APPEND).mp4,$(MOV)) 250 | 251 | .PHONY: mp4-concat 252 | mp4-concat: $(FFMPEG) 253 | ifndef DIR 254 | $(error DIR variable must reference the dir where multiple mp4 files are stored) 255 | endif 256 | $(FFMPEG) -f concat \ 257 | -safe 0 \ 258 | -i <(for f in $(DIR)/*.mp4; do echo "file '$$f'"; done) \ 259 | -c copy \ 260 | $(DIR)/concat.mp4 261 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Official repository for TGI RabbitMQ, a series of videos produced by the RabbitMQ team. 2 | 3 | This show is inspired by [TGI Kubernetes](https://github.com/vmware-tanzu/tgik) - thank you! - with a couple of important differences: 4 | 5 | 1. The last Friday of every month we ship a new episode. 6 | 1. Episodes are pre-recorded. We are on the fence regarding live streaming - while we don't rule it out, we pre-record by default. 7 | 1. And, most importantly, this show is about RabbitMQ, not Kubernetes - there will be some K8S, but the focus is RabbitMQ. 8 | 9 | The [playlist index](playlist.md) contains a list of all content. 10 | 11 | The [official YouTube channel can be found here](https://tgi.rabbitmq.com). 12 | 13 | ## Suggest an episode 14 | 15 | If you have an idea for TGIR please [open an issue in the TGIR issue tracker](https://github.com/rabbitmq/tgir/issues/new). 16 | 17 | We pick the episodes as we see fit with what's happening in the community. 18 | 19 | If we select your suggestion we will link back to your original work here! 20 | -------------------------------------------------------------------------------- /playlist.md: -------------------------------------------------------------------------------- 1 | # TGIR Playlist Index 2 | 3 | ## Season 1 4 | 5 | - [S01E01: How to upgrade from RabbitMQ 3.7 to 3.8? January, 2020](s01/e01/README.md) 6 | - [S01E02: Help! RabbitMQ ate my RAM! February, 2020](s01/e02/README.md) 7 | - [S01E03: How to contribute to RabbitMQ? Part 1. - March, 2020](s01/e03/README.md) 8 | - [S01E04: How to contribute to RabbitMQ? Part 2. - April, 2020](s01/e04/README.md) 9 | - [S01E05: I want to run RabbitMQ on K8S - Where do I start? May, 2020](s01/e05/README.md) 10 | - [S01E06: How to run a reliable RabbitMQ on K8S? September, 2020](s01/e06/README.md) 11 | - [S01E07: How to monitor RabbitMQ? October, 2020](s01/e07/README.md) 12 | - [S01E08: Secure public RabbitMQ clusters - November, 2020](s01/e08/README.md) 13 | - [S01E09: Testing RabbitMQ Resiliency with Chaos Mesh - December, 2020](s01/e09/README.md) 14 | -------------------------------------------------------------------------------- /s01/e01/.env: -------------------------------------------------------------------------------- 1 | ../../.env -------------------------------------------------------------------------------- /s01/e01/Makefile: -------------------------------------------------------------------------------- 1 | include ../../Makefile 2 | 3 | NAMESPACE := tgir-s01e01-$(USER) 4 | 5 | # https://hub.docker.com/_/rabbitmq?tab=tags 6 | DOCKER_RABBITMQ_36x_IMAGE = rabbitmq:3.6.16-management 7 | DOCKER_RABBITMQ_37x_IMAGE = rabbitmq:3.7.23-management 8 | DOCKER_RABBITMQ_38x_IMAGE = rabbitmq:3.8.2-management 9 | DOCKER_RABBITMQ_IMAGE ?= $(DOCKER_RABBITMQ_37x_IMAGE) 10 | # https://hub.docker.com/r/pivotalrabbitmq/perf-test/tags 11 | DOCKER_RABBITMQ_PERFTEST_IMAGE := pivotalrabbitmq/perf-test:dev-2020.01.22 12 | 13 | RMQ_NODE ?= 1 14 | RMQ_MANAGEMENT_PORT := 15672 15 | NUMBER_OF_QUEUES_PER_NODE := 4000 16 | QUEUES_FROM := $$((($(RMQ_NODE)-1)*$(NUMBER_OF_QUEUES_PER_NODE)+1)) 17 | QUEUES_TO := $$(($(RMQ_NODE)*$(NUMBER_OF_QUEUES_PER_NODE))) 18 | MAX_MESSAGES_PER_QUEUE := 1000 19 | MESSAGE_SIZE := 1000 20 | 21 | # You may want to overwrite this with your GCP project, e.g. export GCP_PROJECT=my-project-name 22 | GCP_PROJECT ?= cf-rabbitmq-core 23 | # You may want to overwrite this with your preferred GCP zone, e.g. export GCP_ZONE=us-east1-b 24 | GCP_ZONE ?= europe-west2-b 25 | 26 | .PHONY: deps 27 | deps: $(GCLOUD) ## Resolve all dependencies 28 | $(GCLOUD) auth login \ 29 | && $(GCLOUD) config set project $(GCP_PROJECT) \ 30 | && $(GCLOUD) config set compute/zone $(GCP_ZONE) 31 | 32 | instances: $(GCLOUD) ## List all instances 33 | $(GCLOUD) compute instances list --filter='name ~ $(NAMESPACE)' 34 | 35 | # https://cloud.google.com/logging/docs/view/advanced-queries 36 | 37 | logs: $(GCLOUD) 38 | open "https://console.cloud.google.com/logs/viewer?project=$(GCP_PROJECT)&minLogLevel=0&expandAll=false&limitCustomFacetWidth=true&interval=PT1H&advancedFilter=resource.type%3Dgce_instance%0AlogName%3Dprojects%2F$(GCP_PROJECT)%2Flogs%2Fcos_containers%0ANOT%20jsonPayload.message:%22consumer%20latency%22%0ANOT%20jsonPayload.message:%22has%20a%20client-provided%20name%22%0ANOT%20jsonPayload.message:%22authenticated%20and%20granted%20access%22%0ANOT%20jsonPayload.message:%22starting%20producer%22%0ANOT%20jsonPayload.message:%22starting%20consumer%22%0ANOT%20jsonPayload.message:%22accepting%20AMQP%20connection%22" 39 | 40 | logs-lifecyle: $(GCLOUD) 41 | open "https://console.cloud.google.com/logs/viewer?project=$(GCP_PROJECT)&minLogLevel=0&expandAll=false&limitCustomFacetWidth=true&interval=PT1H&advancedFilter=resource.type%3Dgce_instance%0AlogName%3Dprojects%2F$(GCP_PROJECT)%2Flogs%2Fcos_containers%0AjsonPayload.message:%20(%22starting%20rabbitmq%22%20OR%20%22started%22%20OR%20%22stopping%22%20OR%20%22stopped%22%20AND%20NOT%20%22supervisor%22)" 42 | 43 | define GCP_COS_CONTAINER_DEFAULTS 44 | --boot-disk-type=pd-ssd \ 45 | --labels=namespace=$(NAMESPACE) \ 46 | --container-stdin \ 47 | --container-tty 48 | endef 49 | 50 | # https://cloud.google.com/compute/docs/containers/deploying-containers 51 | server: $(GCLOUD) ## Create RabbitMQ node 52 | time $(GCLOUD) compute instances create-with-container $(NAMESPACE)-rmq$(RMQ_NODE)-server \ 53 | $(GCP_COS_CONTAINER_DEFAULTS) \ 54 | --machine-type=n1-standard-8 \ 55 | --create-disk=name=$(NAMESPACE)-rmq$(RMQ_NODE)-server-persistent,size=200GB,type=pd-ssd,auto-delete=yes \ 56 | --container-mount-disk=name=$(NAMESPACE)-rmq$(RMQ_NODE)-server-persistent,mount-path=/var/lib/rabbitmq \ 57 | --container-env RABBITMQ_ERLANG_COOKIE=$(NAMESPACE) \ 58 | --container-image=$(DOCKER_RABBITMQ_IMAGE) 59 | 60 | define RMQ_SERVER_EXTERNAL_IP 61 | $(GCLOUD) compute instances describe $(NAMESPACE)-rmq$(RMQ_NODE)-server \ 62 | --format='get(networkInterfaces[0].accessConfigs[0].natIP)' 63 | endef 64 | management: $(GCLOUD) management-allow ## Open RabbitMQ Management 65 | open http://"$$($(RMQ_SERVER_EXTERNAL_IP))":$(RMQ_MANAGEMENT_PORT) 66 | 67 | management-deny: $(GCLOUD) 68 | $(GCLOUD) compute firewall-rules describe $(USER)-allow-rmq-management \ 69 | && $(GCLOUD) compute firewall-rules delete $(USER)-allow-rmq-management 70 | 71 | management-allow: $(GCLOUD) 72 | $(GCLOUD) compute firewall-rules describe $(USER)-allow-rmq-management \ 73 | || $(GCLOUD) compute firewall-rules create $(USER)-allow-rmq-management \ 74 | --allow=TCP:$(RMQ_MANAGEMENT_PORT) --source-ranges=$(shell curl -s https://api.ipify.org)/32 75 | 76 | server-37x: $(GCLOUD) app-stop ## Run v3.7.x RabbitMQ node 77 | $(GCLOUD) compute instances update-container $(NAMESPACE)-rmq$(RMQ_NODE)-server \ 78 | --container-image=$(DOCKER_RABBITMQ_37x_IMAGE) 79 | 80 | server-38x: $(GCLOUD) app-stop ## Run v3.8.x RabbitMQ node 81 | $(GCLOUD) compute instances update-container $(NAMESPACE)-rmq$(RMQ_NODE)-server \ 82 | --container-image=$(DOCKER_RABBITMQ_38x_IMAGE) 83 | 84 | app-stop: $(GCLOUD) ## Stop rabbit app on RabbitMQ node 85 | time $(GCLOUD) compute ssh $(NAMESPACE)-rmq$(RMQ_NODE)-server -- \ 86 | "docker exec \$$(docker container ls | awk '/rabbitmq/ { print \$$1 }') rabbitmqctl stop_app" 87 | 88 | server-delete: INSTANCE = rmq$(RMQ_NODE)-server 89 | server-delete: _delete ## Delete RabbitMQ node 90 | 91 | server-ssh: INSTANCE = rmq$(RMQ_NODE)-server 92 | server-ssh: _ssh ## SSH to RabbitMQ node 93 | 94 | server-bash: INSTANCE = rmq$(RMQ_NODE)-server 95 | server-bash: _bash ## Open a shell session on RabbitMQ node 96 | 97 | server-ctop: INSTANCE = rmq$(RMQ_NODE)-server 98 | server-ctop: _ctop ## Run ctop on VM that runs RabbitMQ node 99 | 100 | server-htop: INSTANCE = rmq$(RMQ_NODE)-server 101 | server-htop: _htop ## Run htop on VM that runs RabbitMQ node 102 | 103 | backlog: INSTANCE = rmq$(RMQ_NODE)-backlog 104 | backlog: _backlog ## Simulate RabbitMQ message backlog 105 | 106 | backlog-stop: INSTANCE = rmq$(RMQ_NODE)-backlog 107 | backlog-stop: _delete ## Stop RabbitMQ message backlog 108 | 109 | backlog-ctop: INSTANCE = rmq$(RMQ_NODE)-backlog 110 | backlog-ctop: _ctop ## Run ctop on RabbitMQ message backlog 111 | 112 | workload: INSTANCE = rmq$(RMQ_NODE)-workload 113 | workload: _workload ## Simulate RabbitMQ production workload 114 | 115 | workload-stop: INSTANCE = rmq$(RMQ_NODE)-workload 116 | workload-stop: _delete ## Stop RabbitMQ production workload 117 | 118 | workload-ctop: INSTANCE = rmq$(RMQ_NODE)-workload 119 | workload-ctop: _ctop ## Run ctop on RabbitMQ production workload 120 | 121 | drain: INSTANCE = rmq$(RMQ_NODE)-drain 122 | drain: _drain ## Simulate RabbitMQ backlog drain 123 | 124 | drain-stop: INSTANCE = rmq$(RMQ_NODE)-drain 125 | drain-stop: _delete ## Stop RabbitMQ backlog drain 126 | 127 | drain-ctop: INSTANCE = rmq$(RMQ_NODE)-drain 128 | drain-ctop: _ctop ## Run ctop on RabbitMQ backlog drain 129 | 130 | _ssh: $(GCLOUD) 131 | $(GCLOUD) compute ssh $(NAMESPACE)-$(INSTANCE) 132 | 133 | _bash: $(GCLOUD) 134 | $(GCLOUD) compute ssh $(NAMESPACE)-$(INSTANCE) -- \ 135 | "docker exec -it \$$(docker container ls | awk '/rabbitmq/ { print \$$1 }') bash" 136 | 137 | _delete: $(GCLOUD) 138 | time $(GCLOUD) compute instances delete $(NAMESPACE)-$(INSTANCE) 139 | 140 | # https://github.com/bcicen/ctop 141 | define CTOP_CONTAINER 142 | docker run --rm --interactive --tty \ 143 | --cpus 0.5 --memory 128M \ 144 | --volume /var/run/docker.sock:/var/run/docker.sock \ 145 | --name ctop \ 146 | quay.io/vektorlab/ctop 147 | endef 148 | _ctop: $(GCLOUD) 149 | $(GCLOUD) compute ssh $(NAMESPACE)-$(INSTANCE) -- "$(CTOP_CONTAINER)" 150 | 151 | # https://github.com/hishamhm/htop 152 | define HTOP_CONTAINER 153 | docker run --rm --interactive --tty \ 154 | --cpus 0.5 --memory 128M \ 155 | --net="host" --pid="host" \ 156 | --name htop \ 157 | jess/htop 158 | endef 159 | _htop: $(GCLOUD) 160 | $(GCLOUD) compute ssh $(NAMESPACE)-$(INSTANCE) -- "$(HTOP_CONTAINER)" 161 | 162 | define RABBITMQ_PERFTEST_DEFAULTS 163 | --container-image=$(DOCKER_RABBITMQ_PERFTEST_IMAGE) \ 164 | --container-arg="--auto-delete" \ 165 | --container-arg="false" \ 166 | --container-arg="--consumers" \ 167 | --container-arg="$(NUMBER_OF_QUEUES_PER_NODE)" \ 168 | --container-arg="--confirm" \ 169 | --container-arg="1" \ 170 | --container-arg="--confirm-timeout" \ 171 | --container-arg="120" \ 172 | --container-arg="--connection-recovery-interval" \ 173 | --container-arg="240" \ 174 | --container-arg="--flag" \ 175 | --container-arg="persistent" \ 176 | --container-arg="--heartbeat-sender-threads" \ 177 | --container-arg="10" \ 178 | --container-arg="--nio-threads" \ 179 | --container-arg="10" \ 180 | --container-arg="--nio-thread-pool" \ 181 | --container-arg="20" \ 182 | --container-arg="--producers" \ 183 | --container-arg="$(NUMBER_OF_QUEUES_PER_NODE)" \ 184 | --container-arg="--producer-random-start-delay" \ 185 | --container-arg="60" \ 186 | --container-arg="--producer-scheduler-threads" \ 187 | --container-arg="10" \ 188 | --container-arg="--qos" \ 189 | --container-arg="5" \ 190 | --container-arg="--queue-args" \ 191 | --container-arg="x-max-length=$(MAX_MESSAGES_PER_QUEUE)" \ 192 | --container-arg="--queue-pattern" \ 193 | --container-arg="q%d" \ 194 | --container-arg="--queue-pattern-from" \ 195 | --container-arg="$(QUEUES_FROM)" \ 196 | --container-arg="--queue-pattern-to" \ 197 | --container-arg="$(QUEUES_TO)" \ 198 | --container-arg="--servers-startup-timeout" \ 199 | --container-arg="30" \ 200 | --container-arg="--size" \ 201 | --container-arg="$(MESSAGE_SIZE)" \ 202 | --container-arg="--uri" \ 203 | --container-arg="amqp://guest:guest@$(NAMESPACE)-rmq$(RMQ_NODE)-server.c.$(GCP_PROJECT).internal:5672/%2f" 204 | endef 205 | 206 | _backlog: $(GCLOUD) 207 | $(GCLOUD) compute instances create-with-container $(NAMESPACE)-$(INSTANCE) \ 208 | $(GCP_COS_CONTAINER_DEFAULTS) \ 209 | --machine-type=n1-highcpu-4 \ 210 | --container-arg="--consumers" \ 211 | --container-arg="0" \ 212 | --container-arg="--rate" \ 213 | --container-arg="1" \ 214 | $(RABBITMQ_PERFTEST_DEFAULTS) 215 | 216 | _drain: $(GCLOUD) 217 | $(GCLOUD) compute instances create-with-container $(NAMESPACE)-$(INSTANCE) \ 218 | $(GCP_COS_CONTAINER_DEFAULTS) \ 219 | --machine-type=n1-highcpu-4 \ 220 | --container-arg="--autoack" \ 221 | --container-arg="--producers" \ 222 | --container-arg="0" \ 223 | $(RABBITMQ_PERFTEST_DEFAULTS) 224 | 225 | _workload: $(GCLOUD) 226 | $(GCLOUD) compute instances create-with-container $(NAMESPACE)-$(INSTANCE) \ 227 | $(GCP_COS_CONTAINER_DEFAULTS) \ 228 | --machine-type=n1-highcpu-4 \ 229 | --container-arg="--consumer-latency" \ 230 | --container-arg="5000000" \ 231 | --container-arg="--variable-rate" \ 232 | --container-arg="1:60" \ 233 | --container-arg="--variable-rate" \ 234 | --container-arg="0:240" \ 235 | $(RABBITMQ_PERFTEST_DEFAULTS) 236 | -------------------------------------------------------------------------------- /s01/e01/tgir-s01e01-logs-rmq-lifecycle.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rabbitmq/tgir/87c753ae5bffa8cb8b321ca58297858153adeb6f/s01/e01/tgir-s01e01-logs-rmq-lifecycle.png -------------------------------------------------------------------------------- /s01/e01/tgir-s01e01-logs.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rabbitmq/tgir/87c753ae5bffa8cb8b321ca58297858153adeb6f/s01/e01/tgir-s01e01-logs.png -------------------------------------------------------------------------------- /s01/e01/tgir-s01e01-rmq-management-37x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rabbitmq/tgir/87c753ae5bffa8cb8b321ca58297858153adeb6f/s01/e01/tgir-s01e01-rmq-management-37x.png -------------------------------------------------------------------------------- /s01/e01/tgir-s01e01-rmq-management-38x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rabbitmq/tgir/87c753ae5bffa8cb8b321ca58297858153adeb6f/s01/e01/tgir-s01e01-rmq-management-38x.png -------------------------------------------------------------------------------- /s01/e01/video.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rabbitmq/tgir/87c753ae5bffa8cb8b321ca58297858153adeb6f/s01/e01/video.png -------------------------------------------------------------------------------- /s01/e02/.env: -------------------------------------------------------------------------------- 1 | ../../.env -------------------------------------------------------------------------------- /s01/e02/Erlang-Memory-Allocators-100k-messages-lazy-q.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rabbitmq/tgir/87c753ae5bffa8cb8b321ca58297858153adeb6f/s01/e02/Erlang-Memory-Allocators-100k-messages-lazy-q.png -------------------------------------------------------------------------------- /s01/e02/Erlang-Memory-Allocators-50mil-messages-durable-q.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rabbitmq/tgir/87c753ae5bffa8cb8b321ca58297858153adeb6f/s01/e02/Erlang-Memory-Allocators-50mil-messages-durable-q.png -------------------------------------------------------------------------------- /s01/e02/Erlang-Memory-Allocators-50mil-messages-lazy-q.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rabbitmq/tgir/87c753ae5bffa8cb8b321ca58297858153adeb6f/s01/e02/Erlang-Memory-Allocators-50mil-messages-lazy-q.png -------------------------------------------------------------------------------- /s01/e02/Makefile: -------------------------------------------------------------------------------- 1 | include ../../Makefile 2 | 3 | NAMESPACE := tgir-s01e02-$(USER) 4 | 5 | # https://hub.docker.com/_/rabbitmq?tab=tags 6 | # https://hub.docker.com/r/pivotalrabbitmq/rabbitmq-prometheus/tags?page=1&name=3.8.3 7 | DOCKER_RABBITMQ_IMAGE ?= pivotalrabbitmq/rabbitmq-prometheus:3.8.3-alpha.99-2020.02.23 8 | # https://hub.docker.com/r/pivotalrabbitmq/perf-test/tags 9 | DOCKER_RABBITMQ_PERFTEST_IMAGE := pivotalrabbitmq/perf-test:2.10.0 10 | 11 | amqp_PORT := 5672 12 | management_PORT := 15672 13 | prometheus_PORT := 15692 14 | netdata_PORT := 19999 15 | 16 | 30DAYS := $$((30*24*3600*1000)) 17 | 10s_in_micros := $$((10*1000*1000)) 18 | 26KB := $$((26*1000)) 19 | 1MB := $$((1000*1000)) 20 | 100K := 100000 21 | 1M := 1000000 22 | 23 | # https://github.com/rabbitmq/tgir/issues/5#issuecomment-581384417 24 | # IDLE MESSAGES: 25 | # 2 * 12000000 26 | # 5 * 3000000 27 | # 3 * 1000000 28 | # 14 * 500000 29 | # --------------- 30 | # 49000000 31 | # @ 26KB 32 | # --------------- 33 | # 1.215 TB 34 | 35 | # https://cloud.google.com/compute/pricing 36 | 1CPU_4RAM := n1-standard-1 37 | 8CPU_52RAM := n1-highmem-8 38 | 16CPU_104RAM := n1-highmem-16 39 | 40 | durable_backlog_MACHINE_TYPE := $(1CPU_4RAM) 41 | durable_workload_MACHINE_TYPE := $(1CPU_4RAM) 42 | durable_rabbitmq_MACHINE_TYPE := $(16CPU_104RAM) 43 | durable_rabbitmq_MEMORY_HIGH_WATERMARK := 0.8 44 | durable_rabbitmq_DISK_SIZE := 2000GB 45 | durable_QUEUES := 450 46 | durable_PUBLISHERS := $(durable_QUEUES) 47 | durable_PUBLISH_EVERY := 30 48 | durable_CONSUMERS := $(durable_QUEUES) 49 | durable_CONSUMER_PREFETCH := 50 50 | durable_QUEUE_EXPIRES := $(30DAYS) 51 | durable_MESSAGE_TTL := $(30DAYS) 52 | durable_MESSAGE_SIZE := $(26KB) 53 | durable_BACKLOG_QUEUES := 50 54 | durable_MAX_MESSAGES_PER_BACKLOG_QUEUE := $(1M) 55 | 56 | lazy_workload_MACHINE_TYPE := $(1CPU_4RAM) 57 | lazy_rabbitmq_MACHINE_TYPE := $(1CPU_4RAM) 58 | lazy_rabbitmq_MEMORY_HIGH_WATERMARK := 0.8 59 | lazy_rabbitmq_DISK_SIZE := 200GB 60 | lazy_QUEUES := 1 61 | lazy_PUBLISHERS := $(lazy_QUEUES) 62 | lazy_PUBLISH_RATE_1 := 10:1000 63 | lazy_PUBLISH_RATE_2 := 0:100 64 | lazy_MESSAGE_SIZE := $(1MB) 65 | lazy_CONSUMERS := $(lazy_QUEUES) 66 | lazy_CONSUMER_LATENCY := $(10s_in_micros) 67 | lazy_MAX_MESSAGES_PER_QUEUE := $(100K) 68 | 69 | # You may want to overwrite this with your GCP project, e.g. export GCP_PROJECT=my-project-name 70 | GCP_PROJECT ?= cf-rabbitmq-core 71 | # You may want to overwrite this with your preferred GCP zone, e.g. export GCP_ZONE=us-east1-b 72 | GCP_ZONE ?= europe-west2-b 73 | 74 | .PHONY: deps 75 | deps: $(GCLOUD) ## Resolve all dependencies 76 | $(GCLOUD) auth login \ 77 | && $(GCLOUD) config set project $(GCP_PROJECT) \ 78 | && $(GCLOUD) config set compute/zone $(GCP_ZONE) 79 | 80 | instances: $(GCLOUD) ## List all instances 81 | $(GCLOUD) compute instances list --filter='name ~ $(NAMESPACE)' 82 | 83 | # https://cloud.google.com/logging/docs/view/advanced-queries 84 | 85 | logs: $(GCLOUD) 86 | open "https://console.cloud.google.com/logs/viewer?project=$(GCP_PROJECT)&minLogLevel=0&expandAll=false&limitCustomFacetWidth=true&interval=PT1H&advancedFilter=resource.type%3Dgce_instance%0AlogName%3Dprojects%2F$(GCP_PROJECT)%2Flogs%2Fcos_containers%0ANOT%20jsonPayload.message:%22consumer%20latency%22%0ANOT%20jsonPayload.message:%22has%20a%20client-provided%20name%22%0ANOT%20jsonPayload.message:%22authenticated%20and%20granted%20access%22%0ANOT%20jsonPayload.message:%22starting%20producer%22%0ANOT%20jsonPayload.message:%22starting%20consumer%22%0ANOT%20jsonPayload.message:%22accepting%20AMQP%20connection%22" 87 | 88 | define GCP_COS_CONTAINER_DEFAULTS 89 | --boot-disk-type=pd-ssd \ 90 | --labels=namespace=$(NAMESPACE) \ 91 | --container-stdin \ 92 | --container-tty 93 | endef 94 | 95 | # https://cloud.google.com/compute/docs/containers/deploying-containers 96 | # https://cloud.google.com/compute/pricing 97 | rabbitmq-%: $(GCLOUD) ## Create RabbitMQ server 98 | time $(GCLOUD) compute instances create-with-container $(NAMESPACE)-$@ \ 99 | $(GCP_COS_CONTAINER_DEFAULTS) \ 100 | --machine-type=$($*_rabbitmq_MACHINE_TYPE) \ 101 | --create-disk=name=$(NAMESPACE)-$@-persistent,size=$($*_rabbitmq_DISK_SIZE),type=pd-ssd,auto-delete=yes \ 102 | --container-mount-disk=name=$(NAMESPACE)-$@-persistent,mount-path=/var/lib/rabbitmq \ 103 | --container-env RABBITMQ_ERLANG_COOKIE=$(NAMESPACE) \ 104 | --container-env RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS="-rabbit vm_memory_high_watermark $($*_rabbitmq_MEMORY_HIGH_WATERMARK)" \ 105 | --container-image=$(DOCKER_RABBITMQ_IMAGE) \ 106 | || time $(GCLOUD) compute instances update-container $(NAMESPACE)-$@ \ 107 | --container-env RABBITMQ_ERLANG_COOKIE=$(NAMESPACE) \ 108 | --container-env RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS="-rabbit vm_memory_high_watermark $($*_rabbitmq_MEMORY_HIGH_WATERMARK)" \ 109 | --container-image=$(DOCKER_RABBITMQ_IMAGE) 110 | 111 | define EXTERNAL_IP 112 | $(GCLOUD) compute instances describe $(NAMESPACE)-$* \ 113 | --format='get(networkInterfaces[0].accessConfigs[0].natIP)' 114 | endef 115 | 116 | management-%: $(GCLOUD) firewall-allow-management ## Open RabbitMQ Management 117 | open http://$(shell $(EXTERNAL_IP)):$(management_PORT) 118 | 119 | prometheus-%: $(GCLOUD) firewall-allow-prometheus ## Open RabbitMQ Prometheus 120 | open http://$(shell $(EXTERNAL_IP)):$(prometheus_PORT)/metrics 121 | 122 | firewall-allow-%: $(GCLOUD) 123 | $(GCLOUD) compute firewall-rules describe $(USER)-$(shell hostname)-allow-$* \ 124 | || $(GCLOUD) compute firewall-rules create $(USER)-$(shell hostname)-allow-$* \ 125 | --allow=TCP:$($*_PORT) --source-ranges=$(shell curl -s https://api.ipify.org)/32 126 | 127 | firewall-deny-%: $(GCLOUD) 128 | $(GCLOUD) compute firewall-rules describe $(USER)-$(shell hostname)-allow-$* \ 129 | && $(GCLOUD) compute firewall-rules delete $(USER)-$(shell hostname)-allow-$* 130 | 131 | delete-%: $(GCLOUD) ## Delete instance 132 | time $(GCLOUD) compute instances delete $(NAMESPACE)-$* 133 | 134 | ssh-%: $(GCLOUD) ## Open SSH session 135 | $(GCLOUD) compute ssh $(NAMESPACE)-$* 136 | 137 | bash-%: $(GCLOUD) ## Open a shell session on instance 138 | $(GCLOUD) compute ssh $(NAMESPACE)-$* -- \ 139 | "docker exec -it \$$(docker container ls | awk '/$*/ { print \$$1 }') bash" 140 | 141 | # https://github.com/bcicen/ctop 142 | define CTOP_CONTAINER 143 | docker run --rm --interactive --tty \ 144 | --cpus 0.5 --memory 128M \ 145 | --volume /var/run/docker.sock:/var/run/docker.sock \ 146 | --name ctop \ 147 | quay.io/vektorlab/ctop 148 | endef 149 | ctop-%: $(GCLOUD) ## Open ctop session 150 | $(GCLOUD) compute ssh $(NAMESPACE)-$* -- "$(CTOP_CONTAINER)" 151 | 152 | # https://github.com/hishamhm/htop 153 | define HTOP_CONTAINER 154 | docker run --rm --interactive --tty \ 155 | --cpus 0.5 --memory 128M \ 156 | --net="host" --pid="host" \ 157 | --name htop \ 158 | jess/htop 159 | endef 160 | htop-%: $(GCLOUD) ## Open htop session 161 | $(GCLOUD) compute ssh $(NAMESPACE)-$* -- "$(HTOP_CONTAINER)" 162 | 163 | # https://docs.netdata.cloud/packaging/docker/#run-netdata-with-the-docker-command 164 | define NETDATA_CONTAINER 165 | docker container inspect --format '{{.State.Status}}' netdata \ 166 | || docker run --detach --name=netdata \ 167 | --cpus 1 --memory 8G \ 168 | --publish $(netdata_PORT):$(netdata_PORT) \ 169 | --volume /etc/passwd:/host/etc/passwd:ro \ 170 | --volume /etc/group:/host/etc/group:ro \ 171 | --volume /proc:/host/proc:ro \ 172 | --volume /sys:/host/sys:ro \ 173 | --volume /etc/os-release:/host/etc/os-release:ro \ 174 | --cap-add SYS_PTRACE \ 175 | --security-opt apparmor=unconfined \ 176 | netdata/netdata 177 | endef 178 | netdata-%: $(GCLOUD) firewall-allow-netdata ## Start netdata & open dashboard 179 | $(GCLOUD) compute ssh $(NAMESPACE)-$* -- "$(NETDATA_CONTAINER)" \ 180 | && open http://$(shell $(EXTERNAL_IP)):$(netdata_PORT) 181 | 182 | define durable_backlog_PERFTEST_CONFIG 183 | --container-image=$(DOCKER_RABBITMQ_PERFTEST_IMAGE) \ 184 | --container-arg="--auto-delete" --container-arg="false" \ 185 | --container-arg="--confirm" --container-arg="1" \ 186 | --container-arg="--confirm-timeout" --container-arg="120" \ 187 | --container-arg="--consumers" --container-arg="0" \ 188 | --container-arg="--exchange" --container-arg="topic" \ 189 | --container-arg="--flag" --container-arg="persistent" \ 190 | --container-arg="--heartbeat-sender-threads" --container-arg="10" \ 191 | --container-arg="--nio-threads" --container-arg="10" \ 192 | --container-arg="--nio-thread-pool" --container-arg="20" \ 193 | --container-arg="--producers" --container-arg="$(durable_BACKLOG_QUEUES)" \ 194 | --container-arg="--queue-args" --container-arg="x-queue-mode=lazy,x-max-length=$(durable_MAX_MESSAGES_PER_BACKLOG_QUEUE),x-expires=$(durable_QUEUE_EXPIRES),x-message-ttl=$(durable_MESSAGE_TTL)" \ 195 | --container-arg="--queue-pattern" --container-arg="backlog%d" \ 196 | --container-arg="--queue-pattern-from" --container-arg="1" \ 197 | --container-arg="--queue-pattern-to" --container-arg="$(durable_BACKLOG_QUEUES)" \ 198 | --container-arg="--size" --container-arg="$(durable_MESSAGE_SIZE)" \ 199 | --container-arg="--type" --container-arg="topic" \ 200 | --container-arg="--uri" --container-arg="amqp://guest:guest@$(NAMESPACE)-rabbitmq-durable.c.$(GCP_PROJECT).internal:$(amqp_PORT)/%2f" 201 | endef 202 | backlog-%: $(GCLOUD) ## Simulate message backlog 203 | time $(GCLOUD) compute instances create-with-container $(NAMESPACE)-$@ \ 204 | $(GCP_COS_CONTAINER_DEFAULTS) \ 205 | --machine-type=$($*_backlog_MACHINE_TYPE) \ 206 | $($*_backlog_PERFTEST_CONFIG) \ 207 | || time $(GCLOUD) compute instances update-container $(NAMESPACE)-$@ \ 208 | $($*_backlog_PERFTEST_CONFIG) 209 | 210 | define durable_workload_PERFTEST_CONFIG 211 | --container-image=$(DOCKER_RABBITMQ_PERFTEST_IMAGE) \ 212 | --container-arg="--auto-delete" --container-arg="false" \ 213 | --container-arg="--confirm" --container-arg="1" \ 214 | --container-arg="--confirm-timeout" --container-arg="120" \ 215 | --container-arg="--consumers" --container-arg="$(durable_CONSUMERS)" \ 216 | --container-arg="--exchange" --container-arg="topic" \ 217 | --container-arg="--flag" --container-arg="persistent" \ 218 | --container-arg="--heartbeat-sender-threads" --container-arg="10" \ 219 | --container-arg="--nio-threads" --container-arg="10" \ 220 | --container-arg="--nio-thread-pool" --container-arg="20" \ 221 | --container-arg="--producers" --container-arg="$(durable_PUBLISHERS)" \ 222 | --container-arg="--publishing-interval" --container-arg="$(durable_PUBLISH_EVERY)" \ 223 | --container-arg="--qos" --container-arg="$(durable_CONSUMER_PREFETCH)" \ 224 | --container-arg="--queue-args" --container-arg="x-queue-mode=lazy,x-expires=$(durable_QUEUE_EXPIRES),x-message-ttl=$(durable_MESSAGE_TTL)" \ 225 | --container-arg="--queue-pattern" --container-arg="q%d" \ 226 | --container-arg="--queue-pattern-from" --container-arg="1" \ 227 | --container-arg="--queue-pattern-to" --container-arg="$(durable_QUEUES)" \ 228 | --container-arg="--size" --container-arg="$(durable_MESSAGE_SIZE)" \ 229 | --container-arg="--type" --container-arg="topic" \ 230 | --container-arg="--uri" --container-arg="amqp://guest:guest@$(NAMESPACE)-rabbitmq-durable.c.$(GCP_PROJECT).internal:$(amqp_PORT)/%2f" 231 | endef 232 | define lazy_workload_PERFTEST_CONFIG 233 | --container-image=$(DOCKER_RABBITMQ_PERFTEST_IMAGE) \ 234 | --container-arg="--auto-delete" --container-arg="false" \ 235 | --container-arg="--confirm" --container-arg="1" \ 236 | --container-arg="--confirm-timeout" --container-arg="120" \ 237 | --container-arg="--consumers" --container-arg="$(lazy_CONSUMERS)" \ 238 | --container-arg="--consumer-latency" --container-arg="$(lazy_CONSUMER_LATENCY)" \ 239 | --container-arg="--flag" --container-arg="persistent" \ 240 | --container-arg="--heartbeat-sender-threads" --container-arg="10" \ 241 | --container-arg="--nio-threads" --container-arg="10" \ 242 | --container-arg="--nio-thread-pool" --container-arg="20" \ 243 | --container-arg="--producers" --container-arg="$(lazy_PUBLISHERS)" \ 244 | --container-arg="--qos" --container-arg="1" \ 245 | --container-arg="--queue-args" --container-arg="x-queue-mode=lazy,x-max-length=$(lazy_MAX_MESSAGES_PER_QUEUE)" \ 246 | --container-arg="--queue-pattern" --container-arg="q%d" \ 247 | --container-arg="--queue-pattern-from" --container-arg="1" \ 248 | --container-arg="--queue-pattern-to" --container-arg="$(lazy_QUEUES)" \ 249 | --container-arg="--size" --container-arg="$(lazy_MESSAGE_SIZE)" \ 250 | --container-arg="--uri" --container-arg="amqp://guest:guest@$(NAMESPACE)-rabbitmq-lazy.c.$(GCP_PROJECT).internal:$(amqp_PORT)/%2f" \ 251 | --container-arg="--variable-rate" --container-arg="$(lazy_PUBLISH_RATE_1)" \ 252 | --container-arg="--variable-rate" --container-arg="$(lazy_PUBLISH_RATE_2)" 253 | endef 254 | workload-%: $(GCLOUD) ## Simulate workload 255 | time $(GCLOUD) compute instances create-with-container $(NAMESPACE)-$@ \ 256 | $(GCP_COS_CONTAINER_DEFAULTS) \ 257 | --machine-type=$($*_workload_MACHINE_TYPE) \ 258 | $($*_workload_PERFTEST_CONFIG) \ 259 | || time $(GCLOUD) compute instances update-container $(NAMESPACE)-$@ \ 260 | $($*_workload_PERFTEST_CONFIG) 261 | -------------------------------------------------------------------------------- /s01/e02/README.md: -------------------------------------------------------------------------------- 1 | # TGIR S01E02: Help! RabbitMQ ate my RAM! 2 | 3 | * Proposed by [@Farkie](https://github.com/Farkie) via [rabbitmq/tgir#5](https://github.com/rabbitmq/tgir/issues/5) & [@lyrixx](https://github.com/lyrixx) via [rabbitmq/discussions#59](https://github.com/rabbitmq/discussions/issues/59) 4 | * Hosted by [@gerhardlazu](https://twitter.com/gerhardlazu) 5 | * Published on: 2020-02-28 6 | * Video: https://www.youtube.com/watch?v=dkAhsp-Oxf4 7 | 8 | 9 | 10 | In this episode we will try to understand what happens under the Erlang hood when RabbitMQ is using a lot of memory and what we can do about it. 11 | We will be covering the interactions between RabbitMQ, Erlang Memory Allocators and the Linux Resident Set Size memory. 12 | 13 | There is an unexpected twist - a.k.a. The One Hour Gap - as well as 2 short follow-ups to last month's episode. 14 | 15 | As always, you can follow along - all commands are available as make targets in this episode's directory. 16 | 17 | 18 | 19 | ## Timeline 20 | 21 | - [00:00:00](https://www.youtube.com/watch?v=dkAhsp-Oxf4&t=0s) - Today's topic 22 | - [00:02:29](https://www.youtube.com/watch?v=dkAhsp-Oxf4&t=149s) - RabbitMQ-Overview for 50 mil persistent messages in durable queues 23 | - [00:05:38](https://www.youtube.com/watch?v=dkAhsp-Oxf4&t=338s) - Erlang-Memory-Allocators Grafana dashboard 24 | - [00:06:50](https://www.youtube.com/watch?v=dkAhsp-Oxf4&t=410s) - Resident Set Size (RSS) vs Erlang Allocated Used & Unused 25 | - [00:10:00](https://www.youtube.com/watch?v=dkAhsp-Oxf4&t=600s) - The most important Erlang memory allocators: binary, ETS & eheap 26 | - [00:14:00](https://www.youtube.com/watch?v=dkAhsp-Oxf4&t=840s) - Multiblock vs Singleblock Carriers 27 | - [00:16:26](https://www.youtube.com/watch?v=dkAhsp-Oxf4&t=986s) - Why does Allocated Unused memory keep growing? 28 | - [00:19:11](https://www.youtube.com/watch?v=dkAhsp-Oxf4&t=1151s) - Memory & disk considerations for recovering large backlogs of persistent messages 29 | - [00:25:55](https://www.youtube.com/watch?v=dkAhsp-Oxf4&t=1555s) - Memory profile for 50 mil persistent messages in lazy queues 30 | - [00:30:07](https://www.youtube.com/watch?v=dkAhsp-Oxf4&t=1807s) - Recommendations for @Farkie - rabbitmq/tgir#5 31 | - [00:31:54](https://www.youtube.com/watch?v=dkAhsp-Oxf4&t=1914s) - Recommendations for @lyrixx - rabbitmq/discussions#59 32 | - [00:34:34](https://www.youtube.com/watch?v=dkAhsp-Oxf4&t=2074s) - TGIR S01E01 follow-up: RabbitMQ 3.6 to 3.8 in-place upgrade 33 | - [00:36:16](https://www.youtube.com/watch?v=dkAhsp-Oxf4&t=2176s) - TGIR S01E01 follow-up: RabbitMQ node startup with many vhosts 34 | - [00:37:53](https://www.youtube.com/watch?v=dkAhsp-Oxf4&t=2273s) - [Marques Johansson is paying attention](https://twitter.com/gerhardlazu/status/1223289151086350338)! TGIR is also inspired by [TBS](https://www.youtube.com/channel/UC19FgzMBMqBro361HbE46Fw), not just [TGIK](https://www.youtube.com/playlist?list=PL7bmigfV0EqQzxcNpmcdTJ9eFRPBe-iZa) 🙌 35 | 36 | 37 | 38 | ## Links 39 | 40 | - [Linux ate my ram!](https://www.linuxatemyram.com/) 41 | - [RabbitMQ - Monitoring with Prometheus & Grafana](https://www.rabbitmq.com/prometheus.html) 42 | - [The New Metrics System in RabbitMQ 3.8](https://github.com/rabbitmq/tgir/tree/S01E02/s01/e00) 43 | - [RabbitMQ-Overview for 50 mil persistent messages in durable queues](http://w20.gerhard.io:3000/dashboard/snapshot/OXM7DEODM4pWz4rUNTwB5gXof9ejlHBo) (expires in April 2020) 44 | - [Erlang-Memory-Allocators Grafana dashboard](https://grafana.com/grafana/dashboards/11350) 45 | - [Erlang-Memory-Allocators for 50 mil persistent messages in durable queues](http://w20.gerhard.io:3000/dashboard/snapshot/fg89oxOfxUD85BVFM4v6co7rfaMcpDYr) (expires in April 2020) 46 | - [Should a node with 50mil persistent messages of size 26KB require 80GB of RAM during startup?](https://github.com/rabbitmq/rabbitmq-server/issues/2254) 47 | - [RabbitMQ-Overview for 50 mil persistent messages in lazy queues](http://w20.gerhard.io:3000/dashboard/snapshot/KQcMkRoMgy7h9JevMSbSfwFMBkueQDAc) (expires in April 2020) 48 | - [Erlang-Memory-Allocators for 50 mil persistent messages in lazy queues](http://w20.gerhard.io:3000/dashboard/snapshot/rFwhDiv5mCBAAS6sh5na8ZdOtFw2b420) (expires in April 2020) 49 | - [RabbitMQ-Overview for 100k persistent messages in lazy queues](http://w20.gerhard.io:3000/dashboard/snapshot/PlVavo1iLeC9oYYFb0ZSgY54SQhm0I3m) (expires in April 2020) 50 | - [Erlang-Memory-Allocators for 100k persistent messages in lazy queues](http://w20.gerhard.io:3000/dashboard/snapshot/ZT5DWpjyGlbdsGHWLeo3bVaXiXE8RJHP) (expires in April 2020) 51 | - [Erlang - The Memory Subsystem: Stacks, Heaps and Garbage Collection](https://github.com/happi/theBeamBook/blob/master/chapters/memory.asciidoc) 52 | - [Erlang - erts_alloc](http://erlang.org/doc/man/erts_alloc.html) 53 | - [TGIR S01E01 follow-up: RabbitMQ 3.6 to 3.8 in-place upgrade](https://groups.google.com/d/msg/rabbitmq-users/DnFmA54_LJk/NjjCZ2Q3AwAJ) 54 | - [TGIR S01E01 follow-up: RabbitMQ node startup with many vhosts](https://groups.google.com/d/msg/rabbitmq-users/DnFmA54_LJk/zdewyb_HAwAJ) 55 | - [@displague](https://github.com/displague) is paying attention: [TGIR is a combination of TGIK & TBS](https://twitter.com/gerhardlazu/status/1223289151086350338) 56 | 57 | 58 | 59 | ## Screenshots 60 | 61 | 62 | 63 | 64 | Find more screenshots in the directory. 65 | 66 | 67 | 68 | ## Make targets 69 | 70 | ``` 71 | backlog-% Simulate message backlog 72 | bash-% Open a shell session on instance 73 | ctop-% Open ctop session 74 | delete-% Delete instance 75 | deps Resolve all dependencies 76 | htop-% Open htop session 77 | instances List all instances 78 | management-% Open RabbitMQ Management 79 | netdata-% Start netdata & open dashboard 80 | prometheus-% Open RabbitMQ Prometheus 81 | rabbitmq-% Create RabbitMQ server 82 | ssh-% Open SSH session 83 | workload-% Simulate workload 84 | ``` 85 | 86 | To reproduce the first setup you will need a GCP account & Google Cloud SDK, then just run: 87 | 88 | ``` 89 | make deps 90 | make rabbitmq-durable backlog-durable workload-durable 91 | make netdata-rabbitmq-durable 92 | make management-rabbitmq-durable 93 | make ctop-rabbitmq-durable 94 | ``` 95 | 96 | Replace `durable` with `lazy` in the above commands to reproduce the second setup. 97 | 98 | To delete all instances, run: 99 | 100 | ``` 101 | make delete-rabbitmq-durable delete-backlog-durable delete-workload-durable 102 | make delete-rabbitmq-lazy delete-workload-lazy 103 | ``` 104 | -------------------------------------------------------------------------------- /s01/e02/RabbitMQ-Overview-100k-messages-lazy-q.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rabbitmq/tgir/87c753ae5bffa8cb8b321ca58297858153adeb6f/s01/e02/RabbitMQ-Overview-100k-messages-lazy-q.png -------------------------------------------------------------------------------- /s01/e02/RabbitMQ-Overview-50mil-messages-durable-q.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rabbitmq/tgir/87c753ae5bffa8cb8b321ca58297858153adeb6f/s01/e02/RabbitMQ-Overview-50mil-messages-durable-q.png -------------------------------------------------------------------------------- /s01/e02/RabbitMQ-Overview-50mil-messages-lazy-q.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rabbitmq/tgir/87c753ae5bffa8cb8b321ca58297858153adeb6f/s01/e02/RabbitMQ-Overview-50mil-messages-lazy-q.png -------------------------------------------------------------------------------- /s01/e02/video.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rabbitmq/tgir/87c753ae5bffa8cb8b321ca58297858153adeb6f/s01/e02/video.png -------------------------------------------------------------------------------- /s01/e03/.env: -------------------------------------------------------------------------------- 1 | ../../.env -------------------------------------------------------------------------------- /s01/e03/.gitignore: -------------------------------------------------------------------------------- 1 | rabbitmq-public-umbrella 2 | -------------------------------------------------------------------------------- /s01/e03/Makefile: -------------------------------------------------------------------------------- 1 | include ../../Makefile 2 | 3 | JOBS_IN_PARALLEL ?= 4 4 | DOCKER_IMAGE_TAG := 3.9-$(shell date -u +'%Y%m%d') 5 | 6 | ifeq ($(PLATFORM),Darwin) 7 | ERL ?= /usr/local/bin/erl 8 | $(ERL): 9 | brew install erlang 10 | else 11 | ERL ?= /usr/bin/erl 12 | $(ERL): 13 | $(error Please install Erlang 22 or newer) 14 | endif 15 | 16 | ifeq ($(PLATFORM),Darwin) 17 | MIX ?= /usr/local/bin/mix 18 | $(MIX): 19 | brew install elixir 20 | else 21 | MIX ?= /usr/bin/mix 22 | $(MIX): 23 | $(error Please install Elixir 1.9 or newer) 24 | endif 25 | 26 | .PHONY: deps 27 | deps: | $(ERL) $(MIX) rabbitmq-public-umbrella ## Resolve all dependencies 28 | 29 | rabbitmq-public-umbrella: ## Setup a local copy for developing RabbitMQ 30 | git clone https://github.com/rabbitmq/rabbitmq-public-umbrella.git $@ 31 | time make -C $@ up 32 | 33 | .PHONY: which-release-series 34 | which-release-series: | deps ## Show which release series we are currently on 35 | make -C rabbitmq-public-umbrella show-branch 36 | 37 | .PHONY: switch-to-3.8.x 38 | switch-to-3.8.x: | deps ## Switch local copy to v3.8.x 39 | cd rabbitmq-public-umbrella \ 40 | && git checkout v3.8.x \ 41 | && time make up BRANCH=v3.8.x 42 | 43 | .PHONY: switch-to-3.9.x 44 | switch-to-3.9.x: | deps ## Switch local copy to v3.9.x 45 | cd rabbitmq-public-umbrella \ 46 | && git checkout master \ 47 | && time make up BRANCH=master 48 | 49 | .PHONY: dev-server 50 | dev-server: | deps ## Run dev rabbitmq-server 51 | make -C rabbitmq-public-umbrella/deps/rabbit run-broker 52 | 53 | .PHONY: dev-server-all-plugins 54 | dev-server-all-plugins: | deps ## Run dev rabbitmq-server with all plugins 55 | make -C rabbitmq-public-umbrella/deps/rabbitmq_server_release run-broker RABBITMQ_LOGS=- 56 | 57 | .PHONY: show-ctls 58 | show-ctls: | deps ## Show where to find CTLs 59 | ls rabbitmq-public-umbrella/deps/rabbit/sbin/ 60 | 61 | .PHONY: checkout-pr2279 62 | checkout-pr2279: | switch-to-3.9.x ## Checkout the correct source for rabbitmq-server & rabbitmq-common 63 | cd rabbitmq-public-umbrella/deps/rabbit && git checkout startup_memory_fix 64 | cd rabbitmq-public-umbrella/deps/rabbit_common && git checkout worker_pool_dispatch_sync 65 | 66 | .PHONY: docker-image 67 | docker-image: | deps ## Build & publish a Docker image of a custom RabbitMQ build 68 | make -C rabbitmq-public-umbrella/deps/rabbitmq_server_release docker-image PROJECT_VERSION=$(DOCKER_IMAGE_TAG) 69 | 70 | .PHONY: docker-image-run 71 | docker-image-run: ## Run Docker image of our custom RabbitMQ build 72 | docker run -it --rm -p 15673:15672 pivotalrabbitmq/rabbitmq:$(DOCKER_IMAGE_TAG) 73 | -------------------------------------------------------------------------------- /s01/e03/README.md: -------------------------------------------------------------------------------- 1 | # TGIR S01E03: How to contribute to RabbitMQ? Part 1 2 | 3 | * Proposed via [rabbitmq/tgir#4](https://github.com/rabbitmq/tgir/issues/4) 4 | * Hosted by [@gerhardlazu](https://twitter.com/gerhardlazu) 5 | * Published on: 2020-03-27 6 | * Video: https://www.youtube.com/watch?v=EWU7WCqD_YA 7 | 8 | 9 | 10 | This is the first part of a multi-part series on how to contribute to RabbitMQ. 11 | Our contribution will be to verify that a fix is ready to ship in the next RabbitMQ v3.8.4 patch release. 12 | 13 | We will start by getting a local development copy of RabbitMQ up and running. 14 | Next, we will learn about different ways of running RabbitMQ, and how to use the local CLIs. 15 | To wrap-up, we will build a Docker image and verify that the fix is ready to ship in the next public release. 16 | 17 | To get the best out of this episode, I encourage you to follow along. 18 | All commands are available as make targets in this episode's directory. 19 | 20 | 21 | 22 | ## Timeline 23 | 24 | - [00:00:00](https://www.youtube.com/watch?v=EWU7WCqD_YA&t=0s) - Today's topic 25 | - [00:01:56](https://www.youtube.com/watch?v=EWU7WCqD_YA&t=116s) - What's with the episode README & Makefile? 26 | - [00:04:53](https://www.youtube.com/watch?v=EWU7WCqD_YA&t=293s) - How to setup for local RabbitMQ development? 27 | - [00:06:57](https://www.youtube.com/watch?v=EWU7WCqD_YA&t=417s) - What do the master and v3.8.x branch represent? 28 | - [00:11:05](https://www.youtube.com/watch?v=EWU7WCqD_YA&t=665s) - How to run RabbitMQ in local development mode? 29 | - [00:11:43](https://www.youtube.com/watch?v=EWU7WCqD_YA&t=703s) - What installs Erlang & Elixir? 30 | - [00:14:31](https://www.youtube.com/watch?v=EWU7WCqD_YA&t=871s) - Interactive shell on the local RabbitMQ node 31 | - [00:19:01](https://www.youtube.com/watch?v=EWU7WCqD_YA&t=1141s) - Run a local dev copy of RabbitMQ with all plugins enabled and STDOUT debug logging 32 | - [00:24:35](https://www.youtube.com/watch?v=EWU7WCqD_YA&t=1475s) - Where to find rabbitmqctl & friends? 33 | - [00:27:17](https://www.youtube.com/watch?v=EWU7WCqD_YA&t=1638s) - Man pages for rabbitmqctl & friends 34 | - [00:29:02](https://www.youtube.com/watch?v=EWU7WCqD_YA&t=1742s) - The issue we reported in S01E02 35 | - [00:30:53](https://www.youtube.com/watch?v=EWU7WCqD_YA&t=1853s) - Build a Docker image of rabbitmq-server#2279 & rabbitmq-common#368 36 | - [00:35:48](https://www.youtube.com/watch?v=EWU7WCqD_YA&t=2148s) - Run custom RabbitMQ Docker image locally 37 | - [00:39:00](https://www.youtube.com/watch?v=EWU7WCqD_YA&t=2340s) - Verify that custom RabbitMQ Docker image addresses the issue we reported 38 | - [00:46:18](https://www.youtube.com/watch?v=EWU7WCqD_YA&t=2778s) - rabbitmq-server#2254 is ready to ship in RabbitMQ v3.8.4 patch release 39 | - [00:47:36](https://www.youtube.com/watch?v=EWU7WCqD_YA&t=2856s) - In the next episode... 40 | 41 | 42 | 43 | ## Links 44 | 45 | - [Work with ease on multiple RabbitMQ sub-projects, e.g. core broker, plugins and some client libraries](https://github.com/rabbitmq/rabbitmq-public-umbrella) 46 | - [rabbitmq-server issues #2254: Node can use significant (e.g. 80GB) amounts of RAM during startup](https://github.com/rabbitmq/rabbitmq-server/issues/2254) 47 | - [rabbitmq-server PR #2279: Reduce memory usage during startup](https://github.com/rabbitmq/rabbitmq-server/pull/2279) 48 | - [rabbitmq-common PR #368: Add `worker_pool:dispatch_sync` function](https://github.com/rabbitmq/rabbitmq-common/pull/368) 49 | 50 | 51 | 52 | ## Make targets 53 | 54 | ``` 55 | checkout-pr2279 Checkout the correct source for rabbitmq-server & rabbitmq-common 56 | deps Resolve all dependencies 57 | dev-server Run dev rabbitmq-server 58 | dev-server-all-plugins Run dev rabbitmq-server with all plugins 59 | docker-image Build & publish a Docker image of a custom RabbitMQ build 60 | docker-image-run Run Docker image of our custom RabbitMQ build 61 | rabbitmq-public-umbrella Setup a local copy for developing RabbitMQ 62 | show-ctls Show where to find CTLs 63 | switch-to-3.8.x Switch local copy to v3.8.x 64 | switch-to-3.9.x Switch local copy to v3.9.x 65 | which-release-series Show which release series we are currently on 66 | ``` 67 | 68 | 69 | 70 | ## Notes 71 | 72 | ### SETUP RABBITMQ DEV 73 | - rabbitmq-public-umbrella 74 | - What does the master branch represent? 75 | - And the v3.8.x branch? 76 | 77 | ### RUN RABBITMQ DEV 78 | - How to run a local dev copy of RabbitMQ? 79 | - Interactive shell is available by default 80 | - Run a local dev copy of RabbitMQ with all plugins enabled and STDOUT debug logging 81 | - Where to find rabbitmqctl & friends? 82 | 83 | ### BUILD & TEST CUSTOM RABBITMQ 84 | - Build a Docker image of rabbitmq-server#2279 & rabbitmq-common#368 85 | - Close rabbitmq-server#2254 86 | 87 | ### IN THE NEXT EPISODE... 88 | - Preview docs & share improvements 89 | - 1-command RabbitMQ dev env 90 | - Make code changes, preview & test 91 | - Multi-repo pull-request process 92 | -------------------------------------------------------------------------------- /s01/e03/video.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rabbitmq/tgir/87c753ae5bffa8cb8b321ca58297858153adeb6f/s01/e03/video.jpg -------------------------------------------------------------------------------- /s01/e04/.env: -------------------------------------------------------------------------------- 1 | ../../.env -------------------------------------------------------------------------------- /s01/e04/Makefile: -------------------------------------------------------------------------------- 1 | include ../../Makefile 2 | -------------------------------------------------------------------------------- /s01/e04/README.md: -------------------------------------------------------------------------------- 1 | # TGIR S01E04: How to contribute to RabbitMQ? Part 2 2 | 3 | * Proposed via [rabbitmq/tgir#4](https://github.com/rabbitmq/tgir/issues/4) 4 | * Hosted by [@gerhardlazu](https://twitter.com/gerhardlazu) 5 | * Published on: 2020-04-30 6 | * Video: https://www.youtube.com/watch?v=9q-_DLfhfYg 7 | 8 | 9 | 10 | This is the second part on how to contribute to RabbitMQ. 11 | 12 | See how you can contribute a new Erlang/OTP 22 feature to RabbitMQ 3.8 in these three simple steps: 13 | 14 | 1. Add a few lines of code to [rabbitmq-prometheus](https://github.com/rabbitmq/rabbitmq-prometheus) plugin 15 | 2. Update the [Erlang-Distribution](https://grafana.com/grafana/dashboards/11352) Grafana dashboard 16 | 3. Open a pull request 🙌🏻 17 | 18 | The goal of this episode is for **you** to contribute this change 😉 19 | 20 | Notes and links at https://github.com/rabbitmq/tgir/tree/S01E04/s01/e04 21 | 22 | 23 | 24 | ## TIMELINE 25 | 26 | - This is what you will be contributing, and why (00:00) 27 | - All contributions start with an issue (05:20) 28 | - Start your contribution by writing a failing test (08:19) 29 | - Make the test pass locally (12:19) 30 | - Submit your contribution early as a draft pull-request (13:24) 31 | - Run everything locally & capture observations (15:31) 32 | - Mark the pull-request as ready for review (27:39) 33 | - How can you help us accept contributions promptly (28:00) 34 | - How do you know which version of RabbitMQ to expect a contribution in? (30:00) 35 | - Clean state on your local host (31:38) 36 | 37 | 38 | 39 | ## LINKS 40 | 41 | - [erlang/otp #2270](https://github.com/erlang/otp/pull/2270) - [Which releases?](https://github.com/erlang/otp/pull/2270/commits/302840129567426fd882484606bdc27ed3087eca) 42 | - [prometheus.erl #94](https://github.com/deadtrickster/prometheus.erl/pull/94) 43 | - [rabbitmq/rabbitmq-prometheus #39](https://github.com/rabbitmq/rabbitmq-prometheus/issues/39) 44 | - [Erlang-Distribution Grafana Dashboard](https://grafana.com/grafana/dashboards/11352) 45 | - [Erlang.mk - Common Test](https://erlang.mk/guide/ct.html) 46 | - [RabbitMQ Runtime Tuning - Inter-node Communication Buffer Size](https://www.rabbitmq.com/runtime.html#distribution-buffer) 47 | - [Erlang +zdbbl](https://erlang.org/doc/man/erl.html#+zdbbl) 48 | - [RabbitMQ - All release notes](https://github.com/rabbitmq/rabbitmq-website/tree/live/site/release-notes) 49 | -------------------------------------------------------------------------------- /s01/e04/video.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rabbitmq/tgir/87c753ae5bffa8cb8b321ca58297858153adeb6f/s01/e04/video.jpg -------------------------------------------------------------------------------- /s01/e05/.env: -------------------------------------------------------------------------------- 1 | # shellcheck shell=sh 2 | 3 | main() { 4 | if shell_is_bash 5 | then 6 | load_bash_autocomplete 7 | fi 8 | } 9 | 10 | shell_is_bash() { 11 | # we don't want to depend on pgrep, it doesn't ship on macOS out of the box as far as I can remember 12 | # shellcheck disable=SC2009 13 | ps -p $$ -o comm= | grep -q "bash" 14 | } 15 | 16 | load_bash_autocomplete() { 17 | eval "$(make bash-autocomplete)" 18 | } 19 | 20 | main 21 | -------------------------------------------------------------------------------- /s01/e05/.gitignore: -------------------------------------------------------------------------------- 1 | .config 2 | -------------------------------------------------------------------------------- /s01/e05/Makefile: -------------------------------------------------------------------------------- 1 | include ../../Makefile 2 | 3 | NAMESPACE := tgir-s01e05-$(USER) 4 | K8S_REGION ?= lon1 5 | 6 | DOCTL_CONFIG_DIR := $(XDG_CONFIG_HOME)/doctl 7 | DOCTL_CONFIG := $(DOCTL_CONFIG_DIR)/config.yaml 8 | ifneq ($(wildcard $(DOCTL_CONFIG)),) 9 | DIGITALOCEAN_ACCESS_TOKEN := $(shell awk '/access-token/ { print $$2 }' < $(DOCTL_CONFIG)) 10 | export DIGITALOCEAN_ACCESS_TOKEN 11 | endif 12 | KUBECONFIG_DIR := $(XDG_CONFIG_HOME)/kubectl 13 | KUBECONFIG := $(KUBECONFIG_DIR)/config 14 | export KUBECONFIG 15 | 16 | ifeq ($(PLATFORM),Darwin) 17 | DOCTL ?= /usr/local/bin/doctl 18 | $(DOCTL): 19 | brew install doctl 20 | else 21 | DOCTL ?= /usr/bin/doctl 22 | $(DOCTL): 23 | $(error Please install doctl: https://github.com/digitalocean/doctl#installing-doctl) 24 | endif 25 | .PHONY: doctl 26 | doctl: $(DOCTL) 27 | 28 | .PHONY: k9s 29 | k9s: | $(K9S) $(KUBECONFIG) ## Interact with K8S via terminal UI 30 | $(K9S) --namespace default 31 | 32 | $(DOCTL_CONFIG_DIR): 33 | mkdir -p $(@) 34 | $(DOCTL_CONFIG): | $(DOCTL_CONFIG_DIR) $(DOCTL) 35 | $(DOCTL) --config $(DOCTL_CONFIG) auth init 36 | 37 | $(KUBECONFIG_DIR): 38 | mkdir -p $(@) 39 | $(KUBECONFIG): | $(KUBECONFIG_DIR) $(KUBECTL) $(DOCTL) 40 | $(DOCTL) kubernetes cluster kubeconfig save $(NAMESPACE) 41 | 42 | define ENV 43 | export XDG_CONFIG_HOME="$(XDG_CONFIG_HOME)" 44 | export DIGITALOCEAN_ACCESS_TOKEN="$(DIGITALOCEAN_ACCESS_TOKEN)" 45 | export KUBECONFIG="$(KUBECONFIG)" 46 | unalias k 2>/dev/null; alias k=kubectl 47 | unalias d 2>/dev/null; alias d=doctl 48 | unalias m 2>/dev/null; alias m=make 49 | endef 50 | export ENV 51 | .PHONY: env 52 | env:: | $(DOCTL_CONFIG) $(KUBECONFIG_DIR) ## Configure shell env - eval "$(make env)" 53 | @echo "$$ENV" 54 | 55 | .PHONY: k8s 56 | k8s: | $(DOCTL_CONFIG) $(KUBECONFIG_DIR) ## Create K8S cluster 57 | $(DOCTL) kubernetes cluster get $(NAMESPACE) \ 58 | || time $(DOCTL) kubernetes cluster create $(NAMESPACE) \ 59 | --region $(K8S_REGION) \ 60 | --auto-upgrade \ 61 | --maintenance-window saturday=20:00 62 | 63 | .PHONY: k8s-ls 64 | k8s-ls: | $(DOCTL_CONFIG) ## List K8S clusters 65 | $(DOCTL) kubernetes cluster list 66 | 67 | .PHONY: k8s-rm 68 | k8s-rm: | $(DOCTL_CONFIG) ## Delete K8S cluster 69 | $(DOCTL) kubernetes cluster delete $(NAMESPACE) 70 | 71 | .PHONY: k8s-node-sizes 72 | k8s-node-sizes: | $(DOCTL_CONFIG) ## Show all size options for K8S nodes 73 | $(DOCTL) kubernetes options sizes 74 | 75 | .PHONY: k8s-regions 76 | k8s-regions: | $(DOCTL_CONFIG) ## Show all regions where K8S can be deployed 77 | $(DOCTL) kubernetes options regions 78 | 79 | .PHONY: vms 80 | vms: | $(DOCTL_CONFIG) ## Show all VMs (aka Droplets) 81 | $(DOCTL) compute droplet list && echo 82 | 83 | .PHONY: lbs 84 | lbs: | $(DOCTL_CONFIG) ## Show all Load Balancers 85 | $(DOCTL) compute load-balancer list && echo 86 | 87 | .PHONY: vols 88 | vols: | $(DOCTL_CONFIG) ## Show all Volumes 89 | $(DOCTL) compute volume list && echo 90 | 91 | .PHONY: resources 92 | resources: lbs vms vols ## Show all resources 93 | 94 | .PHONY: rabbitmq 95 | rabbitmq: | $(DOCTL_CONFIG) $(KUBECONFIG) ## Deploy the simplest RabbitMQ on K8S 96 | $(KUBECTL) apply --filename $(CURDIR)/k8s/statefulset.yml 97 | 98 | .PHONY: rabbitmq-rm 99 | rabbitmq-rm: | $(DOCTL_CONFIG) $(KUBECONFIG) ## Delete RabbitMQ and all linked resources 100 | $(KUBECTL) delete --ignore-not-found=true --filename $(CURDIR)/k8s 101 | 102 | .PHONY: shell 103 | shell: | $(KUBECONFIG) ## Open shell in RabbitMQ node 104 | $(KUBECTL) exec -it rabbitmq-0 -- /bin/bash 105 | 106 | .PHONY: public 107 | public: | $(KUBECONFIG) ## Make simplest RabbitMQ on K8S public 108 | $(KUBECTL) apply --filename $(CURDIR)/k8s/service.yml 109 | 110 | .PHONY: persistent 111 | persistent: | $(KUBECONFIG) $(YQ) ## Give RabbitMQ on K8S persistent storage 112 | $(KUBECTL) apply --filename $(CURDIR)/k8s/persistentvolumeclaim.yml 113 | $(YQ) merge $(CURDIR)/k8s/statefulset* | $(KUBECTL) apply --filename - 114 | 115 | .PHONY: benchmark 116 | benchmark: | public $(KUBECONFIG) ## Benchmark simplest RabbitMQ on K8S 117 | $(KUBECTL) create --filename $(CURDIR)/k8s/benchmark.job 118 | 119 | .PHONY: all 120 | all: k8s persistent public benchmark ## Create K8S cluster, deploy RabbitMQ, make public & persistent, then benchmark 121 | 122 | .PHONY: clean 123 | clean: rabbitmq-rm k8s-rm ## Delete RabbitMQ and all linked resources, then delete K8S cluster 124 | -------------------------------------------------------------------------------- /s01/e05/README.md: -------------------------------------------------------------------------------- 1 | # TGIR S01E05: I want to run RabbitMQ on K8S - Where do I start? 2 | 3 | > **Warning** 4 | > This example is intended to demonstrate some of the functionality provided by the [RabbitMQ Cluster Operator](https://github.com/rabbitmq/cluster-operator). It is not intended as a guide for running RabbitMQ on Kubernetes. The RabbitMQ team *strongly* recommend using the operator. 5 | 6 | * Proposed via [rabbitmq/tgir#11](https://github.com/rabbitmq/tgir/issues/11) 7 | * Hosted by [@gerhardlazu](https://twitter.com/gerhardlazu) 8 | * Published on: 2020-05-29 9 | * Video: https://www.youtube.com/watch?v=-yU95ocpBYs 10 | 11 | 12 | 13 | You want to run RabbitMQ on Kubernetes (K8S) and would like to know where to start. 14 | 15 | This video will show you how to: 16 | 1. Setup a K8S cluster with one command and deploy RabbitMQ a few minutes later 17 | 2. Keep persistent data safe 18 | 3. Expose RabbitMQ HTTP & AMQP via public IP 19 | 4. Benchmark it 20 | 21 | 22 | 23 | ## MAKE TARGETS 24 | 25 | ``` 26 | all Create K8S cluster, deploy RabbitMQ, make public & persistent, then benchmark 27 | benchmark Benchmark simplest RabbitMQ on K8S 28 | clean Delete RabbitMQ and all linked resources, then delete K8S cluster 29 | env Configure shell env - eval "$(make env)" 30 | k8s Create K8S cluster 31 | k8s-ls List K8S clusters 32 | k8s-node-sizes Show all size options for K8S nodes 33 | k8s-regions Show all regions where K8S can be deployed 34 | k8s-rm Delete K8S cluster 35 | k9s Interact with K8S via terminal UI 36 | lbs Show all Load Balancers 37 | persistent Give RabbitMQ on K8S persistent storage 38 | public Make simplest RabbitMQ on K8S public 39 | rabbitmq Deploy the simplest RabbitMQ on K8S 40 | rabbitmq-rm Delete RabbitMQ and all linked resources 41 | resources Show all resources 42 | shell Open shell in RabbitMQ node 43 | vms Show all VMs (aka Droplets) 44 | vols Show all Volumes 45 | ``` 46 | -------------------------------------------------------------------------------- /s01/e05/k8s/benchmark.job: -------------------------------------------------------------------------------- 1 | # vim: set filetype=yaml : 2 | apiVersion: batch/v1 3 | kind: Job 4 | metadata: 5 | generateName: benchmark- 6 | spec: 7 | template: 8 | spec: 9 | restartPolicy: Never 10 | containers: 11 | - name: perftest 12 | image: pivotalrabbitmq/perf-test:2.11.0 13 | args: 14 | - "--uri" 15 | - "amqp://guest:guest@rabbitmq:5672/%2f" 16 | - "--producers" 17 | - "1" 18 | - "--consumers" 19 | - "2" 20 | - "--flag" 21 | - "persistent" 22 | - "--auto-delete" 23 | - "false" 24 | - "--size" 25 | - "100" 26 | - "--qos" 27 | - "10" 28 | - "--multi-ack-every" 29 | - "5" 30 | - "--confirm" 31 | - "100" 32 | - "--confirm-timeout" 33 | - "1" 34 | - "--queue" 35 | - "benchmark" 36 | - "--routing-key" 37 | - "benchmark" 38 | - "--time" 39 | - "60" 40 | -------------------------------------------------------------------------------- /s01/e05/k8s/persistentvolumeclaim.yml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: PersistentVolumeClaim 3 | metadata: 4 | name: rabbitmq 5 | spec: 6 | accessModes: 7 | - ReadWriteOnce 8 | resources: 9 | requests: 10 | storage: 10Gi 11 | storageClassName: do-block-storage 12 | -------------------------------------------------------------------------------- /s01/e05/k8s/service.yml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: rabbitmq 5 | labels: 6 | app: rabbitmq 7 | spec: 8 | type: LoadBalancer 9 | selector: 10 | app: rabbitmq 11 | ports: 12 | - port: 5672 13 | targetPort: 5672 14 | name: amqp 15 | protocol: TCP 16 | - port: 15672 17 | targetPort: 15672 18 | name: http 19 | protocol: TCP 20 | -------------------------------------------------------------------------------- /s01/e05/k8s/statefulset.volume: -------------------------------------------------------------------------------- 1 | # vim: set filetype=yaml : 2 | spec: 3 | template: 4 | spec: 5 | containers: 6 | - name: rabbitmq 7 | volumeMounts: 8 | - mountPath: "/var/lib/rabbitmq" 9 | name: rabbitmq 10 | volumes: 11 | - name: rabbitmq 12 | persistentVolumeClaim: 13 | claimName: rabbitmq 14 | -------------------------------------------------------------------------------- /s01/e05/k8s/statefulset.yml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: StatefulSet 3 | metadata: 4 | name: rabbitmq 5 | spec: 6 | serviceName: rabbitmq 7 | selector: 8 | matchLabels: 9 | app: rabbitmq 10 | template: 11 | metadata: 12 | labels: 13 | app: rabbitmq 14 | spec: 15 | containers: 16 | - name: rabbitmq 17 | image: rabbitmq:3.8.3-management 18 | ports: 19 | - name: http 20 | protocol: TCP 21 | containerPort: 15672 22 | - name: amqp 23 | protocol: TCP 24 | containerPort: 5672 25 | livenessProbe: 26 | exec: 27 | command: 28 | - "rabbitmq-diagnostics" 29 | - "ping" 30 | initialDelaySeconds: 10 31 | periodSeconds: 30 32 | timeoutSeconds: 15 33 | readinessProbe: 34 | exec: 35 | command: 36 | - "rabbitmq-diagnostics" 37 | - "check_port_connectivity" 38 | initialDelaySeconds: 10 39 | periodSeconds: 30 40 | timeoutSeconds: 15 41 | -------------------------------------------------------------------------------- /s01/e05/video.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rabbitmq/tgir/87c753ae5bffa8cb8b321ca58297858153adeb6f/s01/e05/video.jpg -------------------------------------------------------------------------------- /s01/e06/.env: -------------------------------------------------------------------------------- 1 | ../../.env -------------------------------------------------------------------------------- /s01/e06/.gitignore: -------------------------------------------------------------------------------- 1 | .config 2 | k8s/secret* 3 | -------------------------------------------------------------------------------- /s01/e06/Makefile: -------------------------------------------------------------------------------- 1 | include ../../Makefile 2 | 3 | TGIR := tgir-s01e06 4 | CLUSTER := $(TGIR)-$(USER) 5 | # Find out what versions are available: make k8s-versions 6 | # K8S versions valid at 28 September 2020 7 | # Pick the latest available version for the control plane, but previous version for the nodes so that we can upgrade 8 | CLUSTER_VERSION ?= 1.18.6-gke.4801 9 | CLUSTER_NODE_VERSION ?= 1.17.9-gke.1504 10 | CLUSTE_RELEASES ?= rapid 11 | CLUSTER_NODE_TYPE ?= n2-standard-4 12 | CLUSTER_NODES_PER_ZONE ?= 2 13 | 14 | # You may want to overwrite this with your GCP project, e.g. export GCP_PROJECT=my-project-name 15 | GCP_PROJECT ?= cf-rabbitmq-core 16 | # You may want to overwrite this with your preferred GCP region, e.g. export GCP_REGION=us-east1 17 | GCP_REGION ?= europe-west2 18 | 19 | KUBECONFIG_DIR := $(XDG_CONFIG_HOME)/kubectl 20 | KUBECONFIG := $(KUBECONFIG_DIR)/config 21 | export KUBECONFIG 22 | 23 | RABBITMQ_DEFAULT_USER ?= $(USER) 24 | RABBITMQ_DEFAULT_PASS ?= $(TGIR) 25 | RABBITMQ_ERLANG_COOKIE ?= $(CLUSTER) 26 | 27 | CLOUDSDK_CONFIG := $(XDG_CONFIG_HOME)/gcloud/configurations/config_default 28 | export CLOUDSDK_CONFIG 29 | $(CLOUDSDK_CONFIG): $(GCLOUD) 30 | $(GCLOUD) auth login \ 31 | && $(GCLOUD) config set project $(GCP_PROJECT) \ 32 | && $(GCLOUD) config set compute/region $(GCP_REGION) 33 | 34 | $(KUBECONFIG_DIR): 35 | mkdir -p $(@) 36 | $(KUBECONFIG): | $(KUBECTL) $(KUBECONFIG_DIR) $(CLOUDSDK_CONFIG) 37 | $(GCLOUD) container clusters get-credentials $(CLUSTER) 38 | 39 | .PHONY: k9s 40 | k9s: | $(K9S) $(KUBECONFIG) ## Interact with our K8S cluster via a terminal UI 41 | $(K9S) --all-namespaces 42 | 43 | .PHONY: k9 44 | k9: | $(K9S) $(KUBECONFIG) 45 | $(K9S) --namespace default --headless 46 | 47 | define ENV 48 | export GCP_PROJECT="$(GCP_PROJECT)" 49 | export GCP_REGION="$(GCP_REGION)" 50 | export XDG_CONFIG_HOME="$(XDG_CONFIG_HOME)" 51 | export KUBECONFIG="$(KUBECONFIG)" 52 | export CLOUDSDK_CONFIG="$(CLOUDSDK_CONFIG)" 53 | unalias k 2>/dev/null; alias k=kubectl 54 | unalias m 2>/dev/null; alias m=make 55 | endef 56 | export ENV 57 | .PHONY: env 58 | env:: | $(CLOUDSDK_CONFIG) $(KUBECONFIG_DIR) ## Configure shell env - eval "$(make env)" OR source .env 59 | @echo "$$ENV" 60 | 61 | define LIST_INSTANCES 62 | $(GCLOUD) compute instances list --filter='name ~ $(CLUSTER)' 63 | endef 64 | instances: | $(CLOUDSDK_CONFIG) ## List all instances 65 | $(LIST_INSTANCES) 66 | 67 | watch-instances: | $(CLOUDSDK_CONFIG) ## Watch all instances 68 | watch -c "$(LIST_INSTANCES)" 69 | 70 | watch-nodes: | $(KUBECONFIG) ## Watch all K8S nodes 71 | watch -c "$(KUBECTL) get nodes --output=wide" 72 | 73 | disks: | $(CLOUDSDK_CONFIG) ## List all disks 74 | $(GCLOUD) compute disks list --filter='name ~ $(CLUSTER)' 75 | 76 | .PHONY: k8s-versions 77 | k8s-versions: | $(CLOUDSDK_CONFIG) ## List all available K8S versions on GCP (GKE) 78 | $(GCLOUD) container get-server-config 79 | 80 | .PHONY: k8s 81 | k8s: | $(CLOUDSDK_CONFIG) ## Create a managed K8S cluster on GCP (GKE) - up to 4 minutes 82 | $(GCLOUD) container clusters describe $(CLUSTER) \ 83 | || time $(GCLOUD) container clusters create $(CLUSTER) \ 84 | --release-channel $(CLUSTE_RELEASES) \ 85 | --cluster-version $(CLUSTER_VERSION) \ 86 | --node-version $(CLUSTER_NODE_VERSION) \ 87 | --machine-type $(CLUSTER_NODE_TYPE) \ 88 | --num-nodes $(CLUSTER_NODES_PER_ZONE) \ 89 | --enable-shielded-nodes \ 90 | --disk-type "pd-ssd" \ 91 | --disk-size "100" \ 92 | --enable-ip-alias \ 93 | --enable-autoupgrade \ 94 | --enable-autorepair \ 95 | --max-surge-upgrade $(CLUSTER_NODES_PER_ZONE) \ 96 | --max-unavailable-upgrade 0 \ 97 | --metadata disable-legacy-endpoints=true \ 98 | --no-enable-master-authorized-networks \ 99 | --addons "HorizontalPodAutoscaling,HttpLoadBalancing" 100 | 101 | .PHONY: k8s-help 102 | k8s-help: | $(CLOUDSDK_CONFIG) ## List all options available when creating a managed K8S cluster on GCP (GKE) 103 | $(GCLOUD) container clusters create --help 104 | 105 | .PHONY: k8s-ls 106 | k8s-ls: | $(CLOUDSDK_CONFIG) ## List all GKE clusters running on GCP 107 | $(GCLOUD) container clusters list 108 | 109 | .PHONY: k8s-rm 110 | k8s-rm: | $(CLOUDSDK_CONFIG) ## Delete our GKE cluster 111 | $(GCLOUD) container clusters delete $(CLUSTER) 112 | 113 | .PHONY: generate-secrets 114 | generate-secrets:: 115 | 116 | k8s/secret.default-user.yml: $(KUBECTL) 117 | $(KUBECTL) create secret generic default-user \ 118 | --dry-run=client --output=yaml --from-literal=value="default_user = $(RABBITMQ_DEFAULT_USER)" > $(@) 119 | generate-secrets:: k8s/secret.default-user.yml 120 | 121 | k8s/secret.default-pass.yml: $(KUBECTL) 122 | $(KUBECTL) create secret generic default-pass \ 123 | --dry-run=client --output=yaml --from-literal=value="default_pass = $(RABBITMQ_DEFAULT_PASS)" > $(@) 124 | generate-secrets:: k8s/secret.default-pass.yml 125 | 126 | k8s/secret.erlang-cookie.yml: $(KUBECTL) 127 | $(KUBECTL) create secret generic erlang-cookie \ 128 | --dry-run=client --output=yaml --from-literal=value="$(RABBITMQ_ERLANG_COOKIE)" > $(@) 129 | generate-secrets:: k8s/secret.erlang-cookie.yml 130 | 131 | .PHONY: rabbitmq 132 | rabbitmq: | $(KUBECONFIG) generate-secrets ## Deploy a reliable RabbitMQ cluster on GKE 133 | $(KUBECTL) apply --filename $(CURDIR)/k8s 134 | 135 | .PHONY: rabbitmq-clients 136 | rabbitmq-clients: | $(KUBECONFIG) generate-secrets ## Deploy reliable RabbitMQ clients on our K8S cluster running in GCP 137 | $(KUBECTL) apply --filename $(CURDIR)/k8s/deployment.consumer \ 138 | && $(KUBECTL) apply --filename $(CURDIR)/k8s/deployment.publisher 139 | 140 | define SERVICE_IP 141 | $(KUBECTL) get service reliable-rabbit-public -o jsonpath="{.status.loadBalancer.ingress[0].ip}" 142 | endef 143 | .PHONY: rabbitmq-management 144 | rabbitmq-management: | $(KUBECONFIG) ## Open RabbitMQ Management in a browser 145 | @while sleep 1; do \ 146 | if [ "$$($(SERVICE_IP) 2>/dev/null)" != "" ]; then \ 147 | break; \ 148 | else \ 149 | echo "Waiting for public IP to be assigned to our RabbitMQ deployment..."; \ 150 | fi \ 151 | done 152 | open http://$$($(SERVICE_IP)):15672 153 | 154 | .PHONY: k8s-operations 155 | k8s-operations: | $(CLOUDSDK_CONFIG) $(BAT) 156 | $(GCLOUD) container operations list --format json \ 157 | | $(BAT) --language json 158 | 159 | .PHONY: k8s-upgrade 160 | k8s-upgrade: | $(CLOUDSDK_CONFIG) ## Upgrade node pool to control plane version - up to 10 minutes 161 | time $(GCLOUD) container clusters upgrade $(CLUSTER) 162 | 163 | .PHONY: rabbitmq-upgrade 164 | rabbitmq-upgrade: | $(YQ) $(KUBECONFIG) ## Upgrade RabbitMQ 165 | $(YQ) merge $(CURDIR)/k8s/statefulset.$(@) \ 166 | $(CURDIR)/k8s/statefulset.yml \ 167 | | $(KUBECTL) apply --filename - 168 | 169 | .PHONY: k8s-force-repair 170 | k8s-force-repair: | $(CLOUDSDK_CONFIG) 171 | $(GCLOUD) container clusters resize $(CLUSTER) \ 172 | --node-pool default-pool \ 173 | --num-nodes $(CLUSTER_NODES_PER_ZONE) 174 | 175 | define INSTANCES_TO_DELETE 176 | $(GCLOUD) compute instances list \ 177 | --filter "zone:($(ZONES)) AND name ~ $(CLUSTER)" --format "value(name,zone)" 178 | endef 179 | .PHONY: _delete_zone 180 | _delete_zone: | $(CLOUDSDK_CONFIG) 181 | $(INSTANCES_TO_DELETE) \ 182 | | awk '{ system("$(GCLOUD) compute instances delete "$$1" --zone="$$2" --quiet 1>/dev/null &") }' 183 | 184 | .PHONY: simulate-loss-of-minority 185 | simulate-loss-of-minority: ZONES = $(GCP_REGION)-a 186 | simulate-loss-of-minority: | _delete_zone ## Simulate losing all instances in 1 zone (minority) 187 | 188 | .PHONY: simulate-loss-of-majority 189 | simulate-loss-of-majority: ZONES = $(GCP_REGION)-a $(GCP_REGION)-b 190 | simulate-loss-of-majority: | _delete_zone ## Simulate losing all instances in 2 zones (majority) 191 | 192 | .PHONY: 193 | simulate-loss-of-all: ZONES = $(GCP_REGION)-a $(GCP_REGION)-b $(GCP_REGION)-c 194 | simulate-loss-of-all: | _delete_zone ## Simulate losing all instances across all 3 zones 195 | 196 | .PHONY: rabbitmq-clients-rm 197 | rabbitmq-clients-rm: | $(KUBECONFIG) ## Delete all RabbitMQ clients 198 | $(KUBECTL) delete --filename $(CURDIR)/k8s/deployment.publisher --wait \ 199 | ; $(KUBECTL) delete --filename $(CURDIR)/k8s/deployment.consumer --wait \ 200 | ; true 201 | 202 | .PHONY: rabbitmq-rm 203 | rabbitmq-rm: | $(KUBECONFIG) ## Delete the RabbitMQ cluster and all associated resources that we have deployed 204 | $(KUBECTL) delete --filename $(CURDIR)/k8s --force --wait \ 205 | ; $(KUBECTL) delete pvc --all --force --wait \ 206 | ; true 207 | 208 | .PHONY: all 209 | all: k8s rabbitmq ## Create K8S cluster & deploy RabbitMQ 210 | 211 | .PHONY: clean 212 | clean: rabbitmq-clients-rm rabbitmq-rm k8s-rm ## Delete the RabbitMQ cluster and all associated resources, then delete the K8S cluster on GKE that we have deployed 213 | -------------------------------------------------------------------------------- /s01/e06/README.md: -------------------------------------------------------------------------------- 1 | # TGIR S01E06: How to run a reliable RabbitMQ on K8S 2 | 3 | 4 | > **Warning** 5 | > This example is intended to demonstrate some of the functionality provided by the [RabbitMQ Cluster Operator](https://github.com/rabbitmq/cluster-operator). It is not intended as a guide for running RabbitMQ on Kubernetes. The RabbitMQ team *strongly* recommend using the operator. 6 | 7 | * Proposed via [rabbitmq/tgir#13](https://github.com/rabbitmq/tgir/issues/13) 8 | * Hosted by [@gerhardlazu](https://twitter.com/gerhardlazu) 9 | * Published on: 2020-09-30 10 | * Video: https://www.youtube.com/watch?v=I02oKJlOnR4 11 | 12 | 13 | 14 | You have a single RabbitMQ node running on Kubernetes (K8S). 15 | [S01E05](https://github.com/rabbitmq/tgir/tree/S01E05/s01/e05) covered the getting started part well. 16 | [Deploying RabbitMQ to Kubernetes: What’s Involved?](https://www.rabbitmq.com/blog/2020/08/10/deploying-rabbitmq-to-kubernetes-whats-involved/) blog post added more detail. 17 | 18 | With the RabbitMQ on K8S basics understood, it's time to deploy a RabbitMQ cluster and tackle more advanced topics: 19 | 20 | 1. What are good liveness & readiness probes? 21 | 2. How to configure RabbitMQ for availability during RabbitMQ upgrades? 22 | 3. How to configure RabbitMQ for availability during K8S upgrades? 23 | 4. How to configure clients for handling a minority of RabbitMQ nodes becoming unavailable? 24 | 5. What to expect when a majority of RabbitMQ nodes go away? 25 | 6. What happens when all RabbitMQ nodes go away? 26 | 27 | 28 | 29 | ## MAKE TARGETS 30 | 31 | ``` 32 | all Create K8S cluster & deploy RabbitMQ 33 | clean Delete the RabbitMQ cluster and all associated resources, then delete the K8S cluster on GKE that we have deployed 34 | disks List all disks 35 | env Configure shell env - eval "$(make env)" OR source .env 36 | instances List all instances 37 | k8s Create a managed K8S cluster on GCP (GKE) - up to 4 minutes 38 | k8s-help List all options available when creating a managed K8S cluster on GCP (GKE) 39 | k8s-ls List all GKE clusters running on GCP 40 | k8s-rm Delete our GKE cluster 41 | k8s-upgrade Upgrade node pool to control plane version - up to 10 minutes 42 | k8s-versions List all available K8S versions on GCP (GKE) 43 | k9s Interact with our K8S cluster via a terminal UI 44 | rabbitmq Deploy a reliable RabbitMQ cluster on GKE 45 | rabbitmq-clients Deploy reliable RabbitMQ clients on our K8S cluster running in GCP 46 | rabbitmq-clients-rm Delete all RabbitMQ clients 47 | rabbitmq-management Open RabbitMQ Management in a browser 48 | rabbitmq-rm Delete the RabbitMQ cluster and all associated resources that we have deployed 49 | rabbitmq-upgrade Upgrade RabbitMQ 50 | simulate-loss-of-all Simulate losing all instances across all 3 zones 51 | simulate-loss-of-majority Simulate losing all instances in 2 zones (majority) 52 | simulate-loss-of-minority Simulate losing all instances in 1 zone (minority) 53 | watch-instances Watch all instances 54 | watch-nodes Watch all K8S nodes 55 | ``` 56 | -------------------------------------------------------------------------------- /s01/e06/k8s/configmap.cluster-formation.yml: -------------------------------------------------------------------------------- 1 | # Configure RabbitMQ cluster formation to use K8S 2 | # 3 | # https://www.rabbitmq.com/cluster-formation.html 4 | apiVersion: v1 5 | kind: ConfigMap 6 | metadata: 7 | name: cluster-formation 8 | data: 9 | value: | 10 | cluster_formation.peer_discovery_backend = k8s 11 | cluster_formation.k8s.host = kubernetes.default.svc.cluster.local 12 | cluster_formation.k8s.address_type = hostname 13 | cluster_formation.k8s.service_name = reliable-rabbit 14 | -------------------------------------------------------------------------------- /s01/e06/k8s/configmap.cluster-name.yml: -------------------------------------------------------------------------------- 1 | # Set RabbitMQ cluster name to a short & friendly name 2 | # 3 | # https://www.rabbitmq.com/configure.html 4 | apiVersion: v1 5 | kind: ConfigMap 6 | metadata: 7 | name: cluster-name 8 | data: 9 | value: | 10 | cluster_name = reliable-rabbit 11 | -------------------------------------------------------------------------------- /s01/e06/k8s/configmap.disk-free-limit.yml: -------------------------------------------------------------------------------- 1 | # Change the default 50MB free disk limit alarm to a production-safe value, relative to the memory available 2 | # 3 | # https://www.rabbitmq.com/configure.html 4 | apiVersion: v1 5 | kind: ConfigMap 6 | metadata: 7 | name: disk-free-limit 8 | data: 9 | value: | 10 | disk_free_limit.relative = 1.0 11 | -------------------------------------------------------------------------------- /s01/e06/k8s/configmap.enabled-plugins.yml: -------------------------------------------------------------------------------- 1 | # Enable specific RabbitMQ plugins 2 | # 3 | # https://www.rabbitmq.com/plugins.html 4 | # 5 | # rabbitmq@rabbitmq-0:/$ rabbitmq-plugins list 6 | # Listing plugins with pattern ".*" ... 7 | # Configured: E = explicitly enabled; e = implicitly enabled 8 | # | Status: * = running on rabbit@rabbitmq-0.reliable-rabbit.default 9 | # |/ 10 | # [ ] rabbitmq_amqp1_0 3.8.9 11 | # [ ] rabbitmq_auth_backend_cache 3.8.9 12 | # [ ] rabbitmq_auth_backend_http 3.8.9 13 | # [ ] rabbitmq_auth_backend_ldap 3.8.9 14 | # [ ] rabbitmq_auth_backend_oauth2 3.8.9 15 | # [ ] rabbitmq_auth_mechanism_ssl 3.8.9 16 | # [ ] rabbitmq_consistent_hash_exchange 3.8.9 17 | # [ ] rabbitmq_event_exchange 3.8.9 18 | # [ ] rabbitmq_federation 3.8.9 19 | # [ ] rabbitmq_federation_management 3.8.9 20 | # [ ] rabbitmq_jms_topic_exchange 3.8.9 21 | # [E*] rabbitmq_management 3.8.9 22 | # [e*] rabbitmq_management_agent 3.8.9 23 | # [ ] rabbitmq_mqtt 3.8.9 24 | # [ ] rabbitmq_peer_discovery_aws 3.8.9 25 | # [e*] rabbitmq_peer_discovery_common 3.8.9 26 | # [ ] rabbitmq_peer_discovery_consul 3.8.9 27 | # [ ] rabbitmq_peer_discovery_etcd 3.8.9 28 | # [E*] rabbitmq_peer_discovery_k8s 3.8.9 29 | # [E*] rabbitmq_prometheus 3.8.9 30 | # [ ] rabbitmq_random_exchange 3.8.9 31 | # [ ] rabbitmq_recent_history_exchange 3.8.9 32 | # [ ] rabbitmq_sharding 3.8.9 33 | # [ ] rabbitmq_shovel 3.8.9 34 | # [ ] rabbitmq_shovel_management 3.8.9 35 | # [ ] rabbitmq_stomp 3.8.9 36 | # [ ] rabbitmq_top 3.8.9 37 | # [ ] rabbitmq_tracing 3.8.9 38 | # [ ] rabbitmq_trust_store 3.8.9 39 | # [e*] rabbitmq_web_dispatch 3.8.9 40 | # [ ] rabbitmq_web_mqtt 3.8.9 41 | # [ ] rabbitmq_web_mqtt_examples 3.8.9 42 | # [ ] rabbitmq_web_stomp 3.8.9 43 | # [ ] rabbitmq_web_stomp_examples 3.8.9 44 | apiVersion: v1 45 | kind: ConfigMap 46 | metadata: 47 | name: enabled-plugins 48 | data: 49 | value: | 50 | [rabbitmq_peer_discovery_k8s, rabbitmq_management, rabbitmq_prometheus]. 51 | -------------------------------------------------------------------------------- /s01/e06/k8s/configmap.log.yml: -------------------------------------------------------------------------------- 1 | # Set the log level to debug, for extra insights 2 | # 3 | # https://www.rabbitmq.com/configure.html 4 | apiVersion: v1 5 | kind: ConfigMap 6 | metadata: 7 | name: log 8 | data: 9 | value: | 10 | log.file.level = debug 11 | -------------------------------------------------------------------------------- /s01/e06/k8s/configmap.queue-master-locator.yml: -------------------------------------------------------------------------------- 1 | # Place newly defined queues on RabbitMQ nodes with the least number of queues 2 | # This ensures that queue-related load is distributed evenly across the cluster 3 | # 4 | # https://www.rabbitmq.com/ha.html#queue-master-location 5 | # https://www.rabbitmq.com/configure.html 6 | apiVersion: v1 7 | kind: ConfigMap 8 | metadata: 9 | name: queue-master-locator 10 | data: 11 | value: | 12 | queue_master_locator = min-masters 13 | -------------------------------------------------------------------------------- /s01/e06/k8s/configmap.total-memory-available-override-value.yml: -------------------------------------------------------------------------------- 1 | # Explicitly define available memory, Erlang/RabbitMQ do not understand cgroups for memory 2 | # Set to the same value as the pod limit 3 | # 4 | # https://www.rabbitmq.com/configure.html 5 | apiVersion: v1 6 | kind: ConfigMap 7 | metadata: 8 | name: total-memory-available-override-value 9 | data: 10 | value: | 11 | total_memory_available_override_value = 12GB 12 | -------------------------------------------------------------------------------- /s01/e06/k8s/configmap.vm-memory-high-watermark-paging-ratio.yml: -------------------------------------------------------------------------------- 1 | # We only use quorum queues, they go straight to disk, paging messages to disk doesn't make sense in this context, disable it 2 | # 3 | # https://www.rabbitmq.com/configure.html 4 | apiVersion: v1 5 | kind: ConfigMap 6 | metadata: 7 | name: vm-memory-high-watermark-paging-ratio 8 | data: 9 | value: | 10 | vm_memory_high_watermark_paging_ratio = 0.99 11 | -------------------------------------------------------------------------------- /s01/e06/k8s/configmap.vm-memory-high-watermark.yml: -------------------------------------------------------------------------------- 1 | # Set the memory alarm to 70% of available memory, it's optimal for this workload 2 | # 3 | # https://www.rabbitmq.com/production-checklist.html#resource-limits-ram 4 | apiVersion: v1 5 | kind: ConfigMap 6 | metadata: 7 | name: vm-memory-high-watermark 8 | data: 9 | value: | 10 | vm_memory_high_watermark.relative = 0.7 11 | -------------------------------------------------------------------------------- /s01/e06/k8s/deployment.consumer: -------------------------------------------------------------------------------- 1 | # vim: set filetype=yaml : 2 | apiVersion: apps/v1 3 | kind: Deployment 4 | metadata: 5 | name: consumer 6 | spec: 7 | # 1-2 per node 8 | replicas: 6 9 | selector: 10 | matchLabels: 11 | app: perftest-consumer 12 | template: 13 | metadata: 14 | labels: 15 | app: perftest-consumer 16 | spec: 17 | # We want to spread clients across all zones, so that if one zone fails, only a minority of clients will be affected (1/3 in this case). 18 | # https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ 19 | topologySpreadConstraints: 20 | - maxSkew: 1 21 | topologyKey: zone 22 | whenUnsatisfiable: ScheduleAnyway 23 | labelSelector: 24 | matchLabels: 25 | app: perftest-consumer 26 | containers: 27 | - name: perftest 28 | image: pivotalrabbitmq/perf-test:2.12.0 29 | # Even if these pods will be scheduled on the same nodes as RabbitMQ, resource limits ensure that they will not compete with RabbitMQ for resources 30 | # https://sysdig.com/blog/kubernetes-limits-requests/ 31 | resources: 32 | limits: 33 | cpu: 1 34 | memory: 1G 35 | requests: 36 | cpu: 500m 37 | memory: 500M 38 | command: 39 | - /bin/bash 40 | - -c 41 | args: 42 | - |- 43 | echo "Use bash string manipulation to extract the values from default-user & default-pass secret confs..." 44 | bin/runjava com.rabbitmq.perf.PerfTest \ 45 | --uri amqp://${RABBITMQ_USER##* }:${RABBITMQ_PASS##* }@$RABBITMQ_SERVICE:5672/%2f 46 | env: 47 | - name: RABBITMQ_USER 48 | valueFrom: 49 | secretKeyRef: 50 | name: default-user 51 | key: value 52 | - name: RABBITMQ_PASS 53 | valueFrom: 54 | secretKeyRef: 55 | name: default-pass 56 | key: value 57 | - name: RABBITMQ_SERVICE 58 | value: reliable-rabbit-public 59 | - name: PRODUCERS 60 | value: "0" 61 | - name: CONSUMERS 62 | value: "1" 63 | - name: CONSUMER_LATENCY 64 | # 10ms 65 | value: "10000" 66 | - name: FLAG 67 | value: "persistent" 68 | - name: AUTO_DELETE 69 | value: "false" 70 | - name: QOS 71 | value: "100" 72 | - name: MULTI_ACK_EVERY 73 | value: "5" 74 | - name: QUEUE 75 | value: "tgir-s01e06" 76 | - name: QUEUE_ARGS 77 | value: "x-queue-type=quorum,x-max-length=1000000" 78 | - name: ROUTING_KEY 79 | value: "tgir-s01e06" 80 | 81 | # http://github.com/rabbitmq/rabbitmq-perf-test 82 | # 83 | # mvnw exec:java -Dexec.mainClass="com.rabbitmq.perf.PerfTest" -Dexec.args="--help --env" 84 | # For multi-value options, separate values with commas, e.g. VARIABLE_RATE='100:60,1000:10,500:15' 85 | # AUTO_DELETE should the queue be auto-deleted, default 86 | # is true 87 | # AUTOACK auto ack 88 | # BODY comma-separated list of files to use in 89 | # message bodies 90 | # BODY_CONTENT_TYPE body content-type 91 | # BODY_COUNT number of pre-generated message bodies. 92 | # Use with --json-body. Default is 100. 93 | # BODY_FIELD_COUNT number of pre-generated fields and values 94 | # for body. Use with --json-body. Default is 95 | # 1000. 96 | # CMESSAGES consumer message count 97 | # CONFIRM max unconfirmed publishes 98 | # CONFIRM_TIMEOUT waiting timeout for unconfirmed publishes 99 | # before failing (in seconds) 100 | # CONNECTION_RECOVERY_INTERVAL connection recovery interval in seconds. 101 | # Default is 5 seconds. Interval syntax, 102 | # e.g. 30-60, is supported to specify an 103 | # random interval between 2 values between 104 | # each attempt. 105 | # CONSUMER_ARGS consumer arguments as key/values pairs, 106 | # separated by commas, e.g. x-priority=10 107 | # CONSUMER_CHANNEL_COUNT channels per consumer 108 | # CONSUMER_LATENCY consumer latency in microseconds 109 | # CONSUMER_RATE consumer rate limit 110 | # CONSUMERS consumer count 111 | # CONSUMERS_THREAD_POOLS number of thread pools to use for all 112 | # consumers, default is to use a thread pool 113 | # for each consumer 114 | # CTXSIZE consumer tx size 115 | # DISABLE_CONNECTION_RECOVERY disable automatic connection recovery 116 | # EXCHANGE exchange name 117 | # EXCLUSIVE use server-named exclusive queues. Such 118 | # queues can only be used by their declaring 119 | # connection! 120 | # FLAG message flag(s), supported values: 121 | # persistent and mandatory. Use the option 122 | # several times to specify several values. 123 | # FRAMEMAX frame max 124 | # GLOBAL_QOS channel prefetch count 125 | # HEARTBEAT heartbeat interval 126 | # HEARTBEAT_SENDER_THREADS number of threads for producers and 127 | # consumers heartbeat senders 128 | # ID test ID 129 | # INTERVAL sampling interval in seconds 130 | # JSON_BODY generate a random JSON document for 131 | # message body. Use with --size. 132 | # LEGACY_METRICS display legacy metrics (min/avg/max 133 | # latency) 134 | # MESSAGE_PROPERTIES message properties as key/value pairs, 135 | # separated by commas, e.g. priority=5 136 | # METRICS_HELP show metrics usage 137 | # MULTI_ACK_EVERY multi ack every 138 | # NACK nack and requeue messages 139 | # NIO_THREAD_POOL size of NIO thread pool, should be 140 | # slightly higher than number of NIO threads 141 | # NIO_THREADS number of NIO threads to use 142 | # OUTPUT_FILE output file for timing results 143 | # PMESSAGES producer message count 144 | # POLLING use basic.get to consume messages. Do not 145 | # use this in real applications. 146 | # POLLING_INTERVAL time to wait before polling with 147 | # basic.get, in millisecond, default is 0. 148 | # PREDECLARED allow use of predeclared objects 149 | # PRODUCER_CHANNEL_COUNT channels per producer 150 | # PRODUCER_RANDOM_START_DELAY max random delay in seconds to start 151 | # producers 152 | # PRODUCER_SCHEDULER_THREADS number of threads to use when using 153 | # --publishing-interval 154 | # PRODUCERS producer count 155 | # PTXSIZE producer tx size 156 | # PUBLISHING_INTERVAL publishing interval in seconds (opposite 157 | # of producer rate limit) 158 | # QOS consumer prefetch count 159 | # QUEUE queue name 160 | # QUEUE_ARGS queue arguments as key/value pairs, 161 | # separated by commas, e.g. x-max-length=10 162 | # QUEUE_PATTERN queue name pattern for creating queues in 163 | # sequence 164 | # QUEUE_PATTERN_FROM queue name pattern range start (inclusive) 165 | # QUEUE_PATTERN_TO queue name pattern range end (inclusive) 166 | # RANDOM_ROUTING_KEY use random routing key per message 167 | # RATE producer rate limit 168 | # ROUTING_KEY routing key 169 | # ROUTING_KEY_CACHE_SIZE size of the random routing keys cache. See 170 | # --random-routing-key. 171 | # SASL_EXTERNAL use SASL EXTERNAL authentication, default 172 | # is false. Set to true if using client 173 | # certificate authentication with the 174 | # rabbitmq_auth_mechanism_ssl plugin. 175 | # SERVERS_STARTUP_TIMEOUT start timeout in seconds (in case the 176 | # servers(s) is (are) not available when the 177 | # run starts). Default is to fail 178 | # immediately if the servers(s) is (are) not 179 | # available. 180 | # SERVERS_UP_LIMIT number of available servers needed before 181 | # starting the run. Used in conjunction with 182 | # --servers-start-timeout. Default is 183 | # deduced from --uri or --uris. 184 | # SHUTDOWN_TIMEOUT shutdown timeout, default is 5 seconds 185 | # SIZE message size in bytes 186 | # SKIP_BINDING_QUEUES don't bind queues to the exchange 187 | # SLOW_START start consumers slowly (1 sec delay 188 | # between each) 189 | # TIME run duration in seconds (unlimited by 190 | # default) 191 | # TYPE exchange type 192 | # URI connection URI 193 | # URIS connection URIs (separated by commas) 194 | # USE_DEFAULT_SSL_CONTEXT use JVM default SSL context 195 | # USE_MILLIS should latency be collected in 196 | # milliseconds, default is false. Set to 197 | # true if producers are consumers run on 198 | # different machines. 199 | # VARIABLE_LATENCY variable consumer processing latency with 200 | # [MICROSECONDS]:[DURATION] syntax, where 201 | # [MICROSECONDS] integer >= 0 and [DURATION] 202 | # integer > 0. Use the option several times 203 | # to specify several values. 204 | # VARIABLE_RATE variable publishing rate with 205 | # [RATE]:[DURATION] syntax, where [RATE] 206 | # integer >= 0 and [DURATION] integer > 0. 207 | # Use the option several times to specify 208 | # several values. 209 | # VARIABLE_SIZE variable message size with 210 | # [SIZE]:[DURATION] syntax, where [SIZE] 211 | # integer > 0 and [DURATION] integer > 0. 212 | # Use the option several times to specify 213 | # several values. 214 | -------------------------------------------------------------------------------- /s01/e06/k8s/deployment.publisher: -------------------------------------------------------------------------------- 1 | # vim: set filetype=yaml : 2 | apiVersion: apps/v1 3 | kind: Deployment 4 | metadata: 5 | name: publisher 6 | spec: 7 | # 1-2 per node 8 | replicas: 6 9 | selector: 10 | matchLabels: 11 | app: perftest-publisher 12 | template: 13 | metadata: 14 | labels: 15 | app: perftest-publisher 16 | spec: 17 | # We want to spread clients across all zones, so that if one zone fails, only a minority of clients will be affected (1/3 in this case). 18 | # https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ 19 | topologySpreadConstraints: 20 | - maxSkew: 1 21 | topologyKey: zone 22 | whenUnsatisfiable: ScheduleAnyway 23 | labelSelector: 24 | matchLabels: 25 | app: perftest-publisher 26 | containers: 27 | - name: perftest 28 | image: pivotalrabbitmq/perf-test:2.12.0 29 | # Even if these pods will be scheduled on the same nodes as RabbitMQ, resource limits ensure that they will not compete with RabbitMQ for resources 30 | # https://sysdig.com/blog/kubernetes-limits-requests/ 31 | resources: 32 | limits: 33 | cpu: 1 34 | memory: 1G 35 | requests: 36 | cpu: 500m 37 | memory: 500M 38 | command: 39 | - /bin/bash 40 | - -c 41 | args: 42 | - |- 43 | echo "Use bash string manipulation to extract the values from default-user & default-pass secret confs..." 44 | bin/runjava com.rabbitmq.perf.PerfTest \ 45 | --uri amqp://${RABBITMQ_USER##* }:${RABBITMQ_PASS##* }@$RABBITMQ_SERVICE:5672/%2f 46 | env: 47 | - name: RABBITMQ_USER 48 | valueFrom: 49 | secretKeyRef: 50 | name: default-user 51 | key: value 52 | - name: RABBITMQ_PASS 53 | valueFrom: 54 | secretKeyRef: 55 | name: default-pass 56 | key: value 57 | - name: RABBITMQ_SERVICE 58 | value: reliable-rabbit-public 59 | - name: PRODUCERS 60 | value: "1" 61 | - name: VARIABLE_RATE 62 | value: "100:60,200:60,50:60,0:60" 63 | - name: CONSUMERS 64 | value: "0" 65 | - name: FLAG 66 | value: "persistent" 67 | - name: AUTO_DELETE 68 | value: "false" 69 | - name: SIZE 70 | value: "1000" 71 | - name: CONFIRM 72 | value: "100" 73 | - name: CONFIRM_TIMEOUT 74 | value: "1" 75 | - name: QUEUE 76 | value: "tgir-s01e06" 77 | - name: QUEUE_ARGS 78 | value: "x-queue-type=quorum,x-max-length=1000000" 79 | - name: ROUTING_KEY 80 | value: "tgir-s01e06" 81 | 82 | # http://github.com/rabbitmq/rabbitmq-perf-test 83 | # 84 | # mvnw exec:java -Dexec.mainClass="com.rabbitmq.perf.PerfTest" -Dexec.args="--help --env" 85 | # For multi-value options, separate values with commas, e.g. VARIABLE_RATE='100:60,1000:10,500:15' 86 | # AUTO_DELETE should the queue be auto-deleted, default 87 | # is true 88 | # AUTOACK auto ack 89 | # BODY comma-separated list of files to use in 90 | # message bodies 91 | # BODY_CONTENT_TYPE body content-type 92 | # BODY_COUNT number of pre-generated message bodies. 93 | # Use with --json-body. Default is 100. 94 | # BODY_FIELD_COUNT number of pre-generated fields and values 95 | # for body. Use with --json-body. Default is 96 | # 1000. 97 | # CMESSAGES consumer message count 98 | # CONFIRM max unconfirmed publishes 99 | # CONFIRM_TIMEOUT waiting timeout for unconfirmed publishes 100 | # before failing (in seconds) 101 | # CONNECTION_RECOVERY_INTERVAL connection recovery interval in seconds. 102 | # Default is 5 seconds. Interval syntax, 103 | # e.g. 30-60, is supported to specify an 104 | # random interval between 2 values between 105 | # each attempt. 106 | # CONSUMER_ARGS consumer arguments as key/values pairs, 107 | # separated by commas, e.g. x-priority=10 108 | # CONSUMER_CHANNEL_COUNT channels per consumer 109 | # CONSUMER_LATENCY consumer latency in microseconds 110 | # CONSUMER_RATE consumer rate limit 111 | # CONSUMERS consumer count 112 | # CONSUMERS_THREAD_POOLS number of thread pools to use for all 113 | # consumers, default is to use a thread pool 114 | # for each consumer 115 | # CTXSIZE consumer tx size 116 | # DISABLE_CONNECTION_RECOVERY disable automatic connection recovery 117 | # EXCHANGE exchange name 118 | # EXCLUSIVE use server-named exclusive queues. Such 119 | # queues can only be used by their declaring 120 | # connection! 121 | # FLAG message flag(s), supported values: 122 | # persistent and mandatory. Use the option 123 | # several times to specify several values. 124 | # FRAMEMAX frame max 125 | # GLOBAL_QOS channel prefetch count 126 | # HEARTBEAT heartbeat interval 127 | # HEARTBEAT_SENDER_THREADS number of threads for producers and 128 | # consumers heartbeat senders 129 | # ID test ID 130 | # INTERVAL sampling interval in seconds 131 | # JSON_BODY generate a random JSON document for 132 | # message body. Use with --size. 133 | # LEGACY_METRICS display legacy metrics (min/avg/max 134 | # latency) 135 | # MESSAGE_PROPERTIES message properties as key/value pairs, 136 | # separated by commas, e.g. priority=5 137 | # METRICS_HELP show metrics usage 138 | # MULTI_ACK_EVERY multi ack every 139 | # NACK nack and requeue messages 140 | # NIO_THREAD_POOL size of NIO thread pool, should be 141 | # slightly higher than number of NIO threads 142 | # NIO_THREADS number of NIO threads to use 143 | # OUTPUT_FILE output file for timing results 144 | # PMESSAGES producer message count 145 | # POLLING use basic.get to consume messages. Do not 146 | # use this in real applications. 147 | # POLLING_INTERVAL time to wait before polling with 148 | # basic.get, in millisecond, default is 0. 149 | # PREDECLARED allow use of predeclared objects 150 | # PRODUCER_CHANNEL_COUNT channels per producer 151 | # PRODUCER_RANDOM_START_DELAY max random delay in seconds to start 152 | # producers 153 | # PRODUCER_SCHEDULER_THREADS number of threads to use when using 154 | # --publishing-interval 155 | # PRODUCERS producer count 156 | # PTXSIZE producer tx size 157 | # PUBLISHING_INTERVAL publishing interval in seconds (opposite 158 | # of producer rate limit) 159 | # QOS consumer prefetch count 160 | # QUEUE queue name 161 | # QUEUE_ARGS queue arguments as key/value pairs, 162 | # separated by commas, e.g. x-max-length=10 163 | # QUEUE_PATTERN queue name pattern for creating queues in 164 | # sequence 165 | # QUEUE_PATTERN_FROM queue name pattern range start (inclusive) 166 | # QUEUE_PATTERN_TO queue name pattern range end (inclusive) 167 | # RANDOM_ROUTING_KEY use random routing key per message 168 | # RATE producer rate limit 169 | # ROUTING_KEY routing key 170 | # ROUTING_KEY_CACHE_SIZE size of the random routing keys cache. See 171 | # --random-routing-key. 172 | # SASL_EXTERNAL use SASL EXTERNAL authentication, default 173 | # is false. Set to true if using client 174 | # certificate authentication with the 175 | # rabbitmq_auth_mechanism_ssl plugin. 176 | # SERVERS_STARTUP_TIMEOUT start timeout in seconds (in case the 177 | # servers(s) is (are) not available when the 178 | # run starts). Default is to fail 179 | # immediately if the servers(s) is (are) not 180 | # available. 181 | # SERVERS_UP_LIMIT number of available servers needed before 182 | # starting the run. Used in conjunction with 183 | # --servers-start-timeout. Default is 184 | # deduced from --uri or --uris. 185 | # SHUTDOWN_TIMEOUT shutdown timeout, default is 5 seconds 186 | # SIZE message size in bytes 187 | # SKIP_BINDING_QUEUES don't bind queues to the exchange 188 | # SLOW_START start consumers slowly (1 sec delay 189 | # between each) 190 | # TIME run duration in seconds (unlimited by 191 | # default) 192 | # TYPE exchange type 193 | # URI connection URI 194 | # URIS connection URIs (separated by commas) 195 | # USE_DEFAULT_SSL_CONTEXT use JVM default SSL context 196 | # USE_MILLIS should latency be collected in 197 | # milliseconds, default is false. Set to 198 | # true if producers are consumers run on 199 | # different machines. 200 | # VARIABLE_LATENCY variable consumer processing latency with 201 | # [MICROSECONDS]:[DURATION] syntax, where 202 | # [MICROSECONDS] integer >= 0 and [DURATION] 203 | # integer > 0. Use the option several times 204 | # to specify several values. 205 | # VARIABLE_RATE variable publishing rate with 206 | # [RATE]:[DURATION] syntax, where [RATE] 207 | # integer >= 0 and [DURATION] integer > 0. 208 | # Use the option several times to specify 209 | # several values. 210 | # VARIABLE_SIZE variable message size with 211 | # [SIZE]:[DURATION] syntax, where [SIZE] 212 | # integer > 0 and [DURATION] integer > 0. 213 | # Use the option several times to specify 214 | # several values. 215 | -------------------------------------------------------------------------------- /s01/e06/k8s/role.yml: -------------------------------------------------------------------------------- 1 | # This is used by rabbitmq-peer-discovery-k8s for cluster formation 2 | # 3 | # https://kubernetes.io/docs/reference/access-authn-authz/rbac/ 4 | kind: Role 5 | apiVersion: rbac.authorization.k8s.io/v1beta1 6 | metadata: 7 | name: rabbitmq 8 | rules: 9 | - apiGroups: 10 | - "" 11 | resources: 12 | - "endpoints" 13 | verbs: 14 | - "get" 15 | - apiGroups: 16 | - "" 17 | resources: 18 | - "events" 19 | verbs: 20 | - "create" 21 | -------------------------------------------------------------------------------- /s01/e06/k8s/rolebinding.yml: -------------------------------------------------------------------------------- 1 | # This is used by rabbitmq-peer-discovery-k8s for cluster formation 2 | # 3 | # https://kubernetes.io/docs/reference/access-authn-authz/rbac/ 4 | kind: RoleBinding 5 | apiVersion: rbac.authorization.k8s.io/v1beta1 6 | metadata: 7 | name: rabbitmq 8 | subjects: 9 | - kind: ServiceAccount 10 | name: rabbitmq 11 | roleRef: 12 | apiGroup: rbac.authorization.k8s.io 13 | kind: Role 14 | name: rabbitmq 15 | -------------------------------------------------------------------------------- /s01/e06/k8s/service.reliable-rabbit-public.yml: -------------------------------------------------------------------------------- 1 | # This is the public service, meaning a load balancer to which RabbitMQ clients connect. 2 | # Notice that we expose all ports for the protocols that RabbitMQ supports, 3 | # including those that require specific rabbitmq plugins to be enabled. 4 | # 5 | # https://kubernetes.io/docs/concepts/services-networking/service/ 6 | kind: Service 7 | apiVersion: v1 8 | metadata: 9 | labels: 10 | app: rabbitmq 11 | type: LoadBalancer 12 | name: reliable-rabbit-public 13 | spec: 14 | # https://www.rabbitmq.com/networking.html 15 | ports: 16 | - name: amqp-ssl 17 | protocol: TCP 18 | port: 5671 19 | - name: amqp 20 | protocol: TCP 21 | port: 5672 22 | - name: http-ssl 23 | protocol: TCP 24 | port: 15671 25 | - name: http 26 | protocol: TCP 27 | port: 15672 28 | - name: mqtt 29 | protocol: TCP 30 | port: 1883 31 | - name: mqtt-ssl 32 | protocol: TCP 33 | port: 8883 34 | - name: mqtt-web 35 | protocol: TCP 36 | port: 15675 37 | - name: prometheus-ssl 38 | protocol: TCP 39 | port: 15691 40 | - name: prometheus 41 | protocol: TCP 42 | port: 15692 43 | - name: stomp 44 | protocol: TCP 45 | port: 61613 46 | - name: stomp-ssl 47 | protocol: TCP 48 | port: 61614 49 | - name: stomp-web 50 | protocol: TCP 51 | port: 15674 52 | 53 | selector: 54 | app: rabbitmq 55 | type: LoadBalancer 56 | -------------------------------------------------------------------------------- /s01/e06/k8s/service.reliable-rabbit.yml: -------------------------------------------------------------------------------- 1 | # This is a headless service, meaning an internal service discovery mechanism. 2 | # It gives identity to RabbitMQ nodes, it is used for cluster formation and inter-node communication. 3 | # 4 | # https://kubernetes.io/docs/concepts/services-networking/service/#headless-services 5 | apiVersion: v1 6 | kind: Service 7 | metadata: 8 | name: reliable-rabbit 9 | spec: 10 | clusterIP: None 11 | ports: 12 | - name: epmd 13 | port: 4369 14 | protocol: TCP 15 | targetPort: 4369 16 | - name: erlang 17 | port: 25672 18 | protocol: TCP 19 | targetPort: 25672 20 | # https://github.com/rabbitmq/cluster-operator/issues/75 21 | publishNotReadyAddresses: true 22 | selector: 23 | app: rabbitmq 24 | type: ClusterIP 25 | -------------------------------------------------------------------------------- /s01/e06/k8s/serviceaccount.yml: -------------------------------------------------------------------------------- 1 | # This is used by rabbitmq-peer-discovery-k8s for cluster formation 2 | # 3 | # https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ 4 | apiVersion: v1 5 | kind: ServiceAccount 6 | metadata: 7 | name: rabbitmq 8 | -------------------------------------------------------------------------------- /s01/e06/k8s/statefulset.rabbitmq-upgrade: -------------------------------------------------------------------------------- 1 | # vim: set filetype=yaml : 2 | spec: 3 | template: 4 | spec: 5 | containers: 6 | - name: rabbitmq 7 | # https://github.com/rabbitmq/rabbitmq-server/releases 8 | image: rabbitmq:3.8.9-management 9 | initContainers: 10 | - name: make-erlang-cookie-only-readable-by-rabbitmq 11 | image: rabbitmq:3.8.9-management 12 | -------------------------------------------------------------------------------- /s01/e06/k8s/storageclass.ssd.yml: -------------------------------------------------------------------------------- 1 | # In order to provision SSD disks for storage, we first need to define the pd-ssd type in GCP 2 | # 3 | # https://cloud.google.com/kubernetes-engine/docs/how-to/persistent-volumes/ssd-pd 4 | apiVersion: storage.k8s.io/v1 5 | kind: StorageClass 6 | metadata: 7 | name: ssd 8 | provisioner: kubernetes.io/gce-pd 9 | parameters: 10 | type: pd-ssd 11 | -------------------------------------------------------------------------------- /s01/e06/video.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rabbitmq/tgir/87c753ae5bffa8cb8b321ca58297858153adeb6f/s01/e06/video.jpg -------------------------------------------------------------------------------- /s01/e07/.env: -------------------------------------------------------------------------------- 1 | ../../.env -------------------------------------------------------------------------------- /s01/e07/.gitignore: -------------------------------------------------------------------------------- 1 | .config 2 | .envrc 3 | bin 4 | tmp 5 | -------------------------------------------------------------------------------- /s01/e07/Makefile: -------------------------------------------------------------------------------- 1 | include $(dir $(word $(words $(MAKEFILE_LIST)),$(MAKEFILE_LIST)))/../../Makefile 2 | 3 | TGIR := tgir-s01e07 4 | 5 | # $ equinix_metal facilities get 6 | DATA_CENTRE ?= am6 7 | # Available bare metal servers: https://metal.equinix.com/product/servers/ 8 | # $ equinix_metal plans get 9 | SERVER_TYPE ?= c3.medium.x86 10 | SERVER_NAME ?= $(TGIR) 11 | # $ equinix_metal operating-systems get 12 | OS ?= ubuntu_20_04 13 | SSH_USER ?= root 14 | 15 | K3S_HOST ?= $(EQUINIX_METAL_SERVER_IP) 16 | 17 | K3S_CHANNEL ?= v1.18 18 | 19 | EQUINIX_METAL_RELEASES := https://github.com/packethost/packet-cli/releases 20 | EQUINIX_METAL_VERSION := 0.1.0 21 | EQUINIX_METAL_BIN := packet-$(EQUINIX_METAL_VERSION)-$(platform)-amd64 22 | EQUINIX_METAL_URL := $(EQUINIX_METAL_RELEASES)/download/$(EQUINIX_METAL_VERSION)/packet-$(platform)-amd64 23 | EQUINIX_METAL := $(LOCAL_BIN)/$(EQUINIX_METAL_BIN) 24 | $(EQUINIX_METAL): | $(CURL) $(LOCAL_BIN) 25 | $(CURL) --progress-bar --fail --location --output $(EQUINIX_METAL) "$(EQUINIX_METAL_URL)" 26 | touch $(EQUINIX_METAL) 27 | chmod +x $(EQUINIX_METAL) 28 | $(EQUINIX_METAL) --version | grep $(EQUINIX_METAL_VERSION) 29 | ln -sf $(EQUINIX_METAL) $(LOCAL_BIN)/equinix_metal 30 | .PHONY: equinix-metal 31 | equinix-metal: $(EQUINIX_METAL) 32 | 33 | K3SUP_RELEASES := https://github.com/alexellis/k3sup/releases 34 | K3SUP_VERSION := 0.9.7 35 | K3SUP_BIN := k3sup-$(K3SUP_VERSION)-$(platform)-amd64 36 | K3SUP_URL := $(K3SUP_RELEASES)/download/$(K3SUP_VERSION)/k3sup-$(platform) 37 | K3SUP := $(LOCAL_BIN)/$(K3SUP_BIN) 38 | $(K3SUP): | $(CURL) $(LOCAL_BIN) 39 | $(CURL) --progress-bar --fail --location --output $(K3SUP) "$(K3SUP_URL)" 40 | touch $(K3SUP) 41 | chmod +x $(K3SUP) 42 | $(K3SUP) version | grep $(K3SUP_VERSION) 43 | ln -sf $(K3SUP) $(LOCAL_BIN)/k3sup 44 | .PHONY: k3sup 45 | k3sup: $(K3SUP) 46 | .PHONY: releases-k3sup 47 | releases-k3sup: 48 | $(OPEN) $(K3SUP_RELEASES) 49 | 50 | KREW_ROOT = $(XDG_CONFIG_HOME)/krew 51 | export KREW_ROOT 52 | KREW_RELEASES := https://github.com/kubernetes-sigs/krew/releases 53 | KREW_VERSION := 0.4.0 54 | KREW_BIN_DIR := krew-$(KREW_VERSION) 55 | KREW_URL := $(KREW_RELEASES)/download/v$(KREW_VERSION)/krew.tar.gz 56 | KREW := $(LOCAL_BIN)/$(KREW_BIN_DIR)/krew-$(platform)_amd64 57 | $(KREW): | $(CURL) $(LOCAL_BIN) 58 | $(CURL) --progress-bar --fail --location --output $(LOCAL_BIN)/$(KREW_BIN_DIR).tar.gz "$(KREW_URL)" 59 | mkdir -p $(LOCAL_BIN)/$(KREW_BIN_DIR) && tar zxf $(LOCAL_BIN)/$(KREW_BIN_DIR).tar.gz -C $(LOCAL_BIN)/$(KREW_BIN_DIR) 60 | touch $(KREW) 61 | chmod +x $(KREW) 62 | $(KREW) version \ 63 | | grep $(KREW_VERSION) 64 | ln -sf $(KREW) $(LOCAL_BIN)/krew 65 | .PHONY: krew 66 | krew: $(KREW) 67 | .PHONY: releases-krew 68 | releases-krew: 69 | $(OPEN) $(KREW_RELEASES) 70 | $(KREW_ROOT)/bin/kubectl-example: $(KREW) 71 | $(KUBECTL) krew update 72 | $(KREW) install example 73 | .PHONY: kubectl-example 74 | kubectl-example: $(KREW_ROOT)/bin/kubectl-example 75 | 76 | KUBECONFIG_DIR := $(XDG_CONFIG_HOME)/kubectl 77 | KUBECONFIG := $(KUBECONFIG_DIR)/config 78 | export KUBECONFIG 79 | $(KUBECONFIG_DIR): 80 | @mkdir -p $(@) 81 | $(KUBECONFIG): | $(K3SUP) $(KUBECTL) $(KUBECONFIG_DIR) 82 | $(K3SUP) install \ 83 | --skip-install \ 84 | --ip $(K3S_HOST) \ 85 | --user $(SSH_USER) \ 86 | --ssh-key $(SSH_PRIVATE_KEY) \ 87 | --local-path $(KUBECONFIG) 88 | .PHONY: kubeconfig 89 | kubeconfig: $(KUBECONFIG) 90 | 91 | .PHONY: k9s 92 | k9s: | $(K9S) $(KUBECONFIG) ## Interact with our K3S instance via a terminal UI 93 | $(K9S) --all-namespaces 94 | 95 | define ENV 96 | export PATH=$(LOCAL_BIN):$(KREW_ROOT)/bin:$$PATH 97 | export XDG_CONFIG_HOME="$(XDG_CONFIG_HOME)" 98 | export KUBECONFIG="$(KUBECONFIG)" 99 | export KREW_ROOT="$(KREW_ROOT)" 100 | unalias k 2>/dev/null; alias k=kubectl 101 | unalias m 2>/dev/null; alias m=make 102 | endef 103 | export ENV 104 | .PHONY: env 105 | env:: | $(KUBECONFIG_DIR) ## Configure shell env - eval "$(make env)" OR source .env 106 | @echo "$$ENV" 107 | 108 | PACKET_TOKEN ?= 109 | PROJECT_ID ?= 110 | .envrc: 111 | @if [[ -z "$(PACKET_TOKEN)" ]] \ 112 | ; then \ 113 | printf "$(BOLD)PACKET_TOKEN$(NORMAL)$(RED) variable must be set so that Equinix Metal CLI operations can run against your Equinix Metal account$(NORMAL)\n" \ 114 | ; exit 1 \ 115 | ; fi 116 | @if [[ -z "$(PROJECT_ID)" ]] \ 117 | ; then \ 118 | printf "$(BOLD)PROJECT_ID$(NORMAL)$(RED) variable must be set so that Equinix Metal CLI operations can run against your Equinix Metal account$(NORMAL)\n" \ 119 | ; exit 1 \ 120 | ; fi 121 | @echo "export PACKET_TOKEN=$(PACKET_TOKEN)" > .envrc 122 | @echo "export PROJECT_ID=$(PACKET_TOKEN)" >> .envrc 123 | 124 | SSH_PUBLIC_KEY ?= 125 | SSH_PUBLIC_KEY_NAME = $(notdir $(basename $(SSH_PUBLIC_KEY))) 126 | .PHONY: equinix-metal-ssh-key 127 | equinix-metal-ssh-key: | $(EQUINIX_METAL) ## Add SSH key that will be added to all new servers 128 | @if [[ ! -r "$(SSH_PUBLIC_KEY)" ]] \ 129 | ; then \ 130 | printf "$(BOLD)SSH_PUBLIC_KEY$(NORMAL)$(RED) variable must be set to a public SSH key file path, e.g. ~/.ssh/id_rsa.pub$(NORMAL)\n" \ 131 | ; exit 1 \ 132 | ; fi 133 | $(EQUINIX_METAL) ssh-key create --key "$$(cat $(SSH_PUBLIC_KEY))" --label $(USER)-$(SSH_PUBLIC_KEY_NAME) 134 | 135 | .PHONY: equinix-metal-server 136 | equinix-metal-server: | $(EQUINIX_METAL) ## Create a Equinix Metal host - optional step 137 | $(EQUINIX_METAL) server get --id $$($(SERVERS) --json 2>/dev/null | $(FIND_SERVER_ID)) \ 138 | || $(EQUINIX_METAL) server create \ 139 | --facility $(DATA_CENTRE) \ 140 | --hostname $(SERVER_NAME) \ 141 | --operating-system $(OS) \ 142 | --plan $(SERVER_TYPE) \ 143 | --project-id $(PROJECT_ID) 144 | 145 | SERVERS = $(EQUINIX_METAL) server get --project-id $(PROJECT_ID) 146 | FIND_SERVER_ID = $(JQ) --raw-output '.[] | select(.hostname == "$(SERVER_NAME)") | .id' 147 | 148 | define EQUINIX_METAL_SERVER_IP 149 | $$($(SERVERS) --json 2>/dev/null \ 150 | | $(JQ) --raw-output '.[] | select(.hostname == "$(SERVER_NAME)") | .ip_addresses[] | select(.address_family == 4) | select(.public == true) | .address') 151 | endef 152 | .PHONY: equinix-metal-server-rm 153 | equinix-metal-server-rm: | $(EQUINIX_METAL) $(JQ) ## Delete the Equinix Metal server - this deletes everything 154 | $(EQUINIX_METAL) server delete --id $$($(SERVERS) --json 2>/dev/null | $(FIND_SERVER_ID)) 155 | 156 | equinix-metal-servers: | $(EQUINIX_METAL) ## List all Equinix Metal servers 157 | $(SERVERS) $(ARGS) 158 | 159 | ssh-k3s: ## SSH into the K3S host 160 | ssh $(SSH_USER)@$(K3S_HOST) 161 | 162 | SSH_PRIVATE_KEY ?= 163 | .PHONY: k3s 164 | k3s: | $(K3SUP) ## Create a K3S instance on a Linux host - works on any Ubuntu VM 165 | @if [[ ! -r "$(SSH_PRIVATE_KEY)" ]] \ 166 | ; then \ 167 | printf "$(BOLD)SSH_PRIVATE_KEY$(NORMAL)$(RED) variable must be set to the private SSH key file path, e.g. ~/.ssh/id_rsa$(NORMAL)\n" \ 168 | ; exit 1 \ 169 | ; fi 170 | $(K3SUP) install \ 171 | --ip $(K3S_HOST) \ 172 | --user $(SSH_USER) \ 173 | --ssh-key $(SSH_PRIVATE_KEY) \ 174 | --local-path $(KUBECONFIG) \ 175 | --k3s-channel $(K3S_CHANNEL) 176 | 177 | # https://github.com/cablespaghetti/k3s-monitoring 178 | # + https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack 179 | # 180 | # Re: manifest_sorter.go:192: info: skipping unknown hook: "crd-install" 181 | # https://github.com/prometheus-community/helm-charts/issues/155#issuecomment-706743032 182 | # 183 | # helm search repo --versions prometheus-community/kube-prometheus-stack 184 | KUBE_PROMETHEUS_STACK_VERSION ?= 10.3.1 185 | .PHONY: k3s-monitoring 186 | k3s-monitoring: | $(KUBECONFIG) $(HELM) ## Integrate Prometheus & Grafana with K3S, including system metrics 187 | $(HELM) repo add prometheus-community https://prometheus-community.github.io/helm-charts 188 | $(HELM) upgrade prometheus prometheus-community/kube-prometheus-stack \ 189 | --install \ 190 | --version $(KUBE_PROMETHEUS_STACK_VERSION) \ 191 | --values k8s/prometheus-stack/values.yml 192 | $(KUBECTL) apply \ 193 | --filename k8s/prometheus-stack/traefik-servicemonitor.yml \ 194 | --filename k8s/prometheus-stack/traefik-grafana-dashboard.yml 195 | 196 | k3s-grafana: | $(KUBECONFIG) ## Access Grafana running in K3S 197 | $(OPEN) http://$(K3S_HOST) 198 | 199 | k3s-rabbitmq: k3s-rabbitmq-grafana k3s-rabbitmq-operator rabbitmq-default-metrics rabbitmq-minimal-metrics rabbitmq-prometheus-metrics rabbitmq-quorum rabbitmq-stream ## Deploy all things RabbitMQ in K3S: operator, clusters, workloads & dashboards 200 | 201 | # https://grafana.com/orgs/rabbitmq 202 | .PHONY: k3s-rabbitmq-grafana 203 | k3s-rabbitmq-grafana: | $(KUBECONFIG) ## Add RabbitMQ Grafana dashboards to Kube Prometheus Stack 204 | $(KUBECTL) apply \ 205 | --filename k8s/prometheus-stack/rabbitmq-overview-grafana-dashboard.yml \ 206 | --filename k8s/prometheus-stack/rabbitmq-quorum-queues-raft-grafana-dashboard.yml \ 207 | --filename k8s/prometheus-stack/rabbitmq-perftest-grafana-dashboard.yml \ 208 | --filename k8s/prometheus-stack/erlang-distribution-grafana-dashboard.yml \ 209 | --filename k8s/prometheus-stack/erlang-memory-allocators-grafana-dashboard.yml 210 | 211 | # https://github.com/rabbitmq/cluster-operator/releases 212 | RABBITMQ_OPERATOR_VERSION ?= 0.48.0 213 | k3s-rabbitmq-operator: | $(KUBECONFIG) ## Install RabbitMQ Cluster Operator into K3S 214 | $(KUBECTL) apply --filename https://github.com/rabbitmq/cluster-operator/releases/download/$(RABBITMQ_OPERATOR_VERSION)/cluster-operator.yml 215 | 216 | .PHONY: rabbitmq-default-metrics 217 | rabbitmq-default-metrics: | $(KUBECONFIG) ## Deploy a RabbitMQ with many Classic Queues, Publishers & Consumers and default Management metrics 218 | $(KUBECTL) apply --filename k8s/rabbitmq/default-metrics.yml 219 | 220 | .PHONY: rabbitmq-minimal-metrics 221 | rabbitmq-minimal-metrics: | $(KUBECONFIG) ## Deploy a RabbitMQ with many Classic Queues, Publishers & Consumers and minimal Management metrics 222 | $(KUBECTL) apply --filename k8s/rabbitmq/minimal-metrics.yml 223 | 224 | .PHONY: rabbitmq-prometheus-metrics 225 | rabbitmq-prometheus-metrics: | $(KUBECONFIG) ## Deploy a RabbitMQ with many Classic Queues, Publishers & Consumers and no Management metrics 226 | $(KUBECTL) apply --filename k8s/rabbitmq/prometheus-metrics.yml 227 | 228 | .PHONY: rabbitmq-quorum 229 | rabbitmq-quorum: | $(KUBECONFIG) ## Deploy a RabbitMQ with a Quorum Queue workload 230 | $(KUBECTL) apply --filename k8s/rabbitmq/quorum.yml 231 | 232 | .PHONY: rabbitmq-stream 233 | rabbitmq-stream: | $(KUBECONFIG) ## Deploy a RabbitMQ with a Stream Queue workload 234 | $(KUBECTL) apply --filename k8s/rabbitmq/stream.yml 235 | 236 | .PHONY: rabbitmq-rm-% 237 | rabbitmq-rm-%: | $(KUBECONFIG) ## Delete one of the deployed RabbitMQ clusters & associated workloads: classic, quorum or stream 238 | $(KUBECTL) delete --filename k8s/rabbitmq/$(*).yml --wait 239 | 240 | .PHONY: tmp 241 | tmp:: 242 | 243 | tmp/k3s-monitoring: 244 | git clone https://github.com/cablespaghetti/k3s-monitoring $(@) 245 | tmp:: tmp/k3s-monitoring 246 | 247 | tmp/prometheus-community-helm-charts: 248 | git clone https://github.com/prometheus-community/helm-charts $(@) 249 | tmp:: tmp/prometheus-community-helm-charts 250 | 251 | -------------------------------------------------------------------------------- /s01/e07/README.md: -------------------------------------------------------------------------------- 1 | # TGIR S01E07: How to monitor RabbitMQ? 2 | 3 | * Proposed via [rabbitmq/tgir#17](https://github.com/rabbitmq/tgir/issues/17) 4 | * Hosted by [@gerhardlazu](https://twitter.com/gerhardlazu) & [@MichalKuratczyk](https://twitter.com/michalkuratczyk) 5 | * Published on: 2020-11-10 6 | * Video: https://www.youtube.com/watch?v=NWISW6AwpOE 7 | 8 | 9 | 10 | You have a few RabbitMQ deployments. How do you monitor them? 11 | 12 | You have heard of the [Grafana dashboards that team RabbitMQ maintains](https://grafana.com/orgs/rabbitmq), maybe from this [RabbitMQ Summit 2019 talk](https://www.youtube.com/watch?v=L-tYXpirbpA) or from the official [Monitoring with Prometheus & Grafana](https://www.rabbitmq.com/prometheus.html) guide. But how do you actually set them up? 13 | 14 | And what about the default metrics configuration? Can this be improved? 15 | 16 | For speed and convenience, we set up K3S on a Linux host ([we had some Equinix Metal credits](https://info.equinixmetal.com/changelog) that we put to good use) and then: 17 | 18 | * We integrate Prometheus & Grafana with K3S, all running inside K3S. 19 | * We set up RabbitMQ Grafana dashboards & deploy the RabbitMQ Cluster Operator, which makes deploying RabbitMQ on K8S as easy as it gets. 20 | * We deploy a few RabbitMQ workloads and look at their behaviour via Grafana, paying special attention to memory pressure coming from the metrics system. 21 | 22 | If your RabbitMQ nodes run many queues, channels & connections and you are using the default metrics configuration, this will help you understand how to optimise that. 23 | 24 | 25 | 26 | ## LINKS 27 | 28 | * [Monitoring with Prometheus & Grafana guide](https://www.rabbitmq.com/prometheus.html) 29 | * [RabbitMQ Cluster Kubernetes Operator](https://github.com/rabbitmq/cluster-operator) 30 | * [RabbitMQ Grafana Dashboards](https://grafana.com/orgs/rabbitmq) 31 | * Observe and understand RabbitMQ - RabbitMQ Summit 2019: [Post](https://www.cloudamqp.com/blog/2019-12-10-observe-and-understand-rabbitmq.html) [Video](https://www.youtube.com/watch?v=L-tYXpirbpA) [Slides](https://gerhard.io/slides/observe-understand-rabbitmq/#/) 32 | * [Equinix Bare Metal - Servers](https://metal.equinix.com/product/servers/) 33 | 34 | 35 | ## MAKE TARGETS 36 | 37 | ``` 38 | env Configure shell env - eval "$(make env)" OR source .env 39 | equinix-metal-server Create a Equinix Metal host - optional step 40 | equinix-metal-server-rm Delete the Equinix Metal server - this deletes everything 41 | equinix-metal-servers List all Equinix Metal servers 42 | equinix-metal-ssh-key Add SSH key that will be added to all new servers 43 | k3s Create a K3S instance on a Linux host - works on any Ubuntu VM 44 | k3s-grafana Access Grafana running in K3S 45 | k3s-monitoring Integrate Prometheus & Grafana with K3S, including system metrics 46 | k3s-rabbitmq Deploy all things RabbitMQ in K3S 47 | k3s-rabbitmq-grafana Add RabbitMQ Grafana dashboards to Kube Prometheus Stack 48 | k3s-rabbitmq-operator Install RabbitMQ Cluster Operator into K3S 49 | k9s Interact with our K3S instance via a terminal UI 50 | rabbitmq-default-metrics Deploy a RabbitMQ with many Classic Queues, Publishers & Consumers and default Management metrics 51 | rabbitmq-minimal-metrics Deploy a RabbitMQ with many Classic Queues, Publishers & Consumers and minimal Management metrics 52 | rabbitmq-prometheus-metrics Deploy a RabbitMQ with many Classic Queues, Publishers & Consumers and no Management metrics 53 | rabbitmq-quorum Deploy a RabbitMQ with a Quorum Queue workload 54 | rabbitmq-rm-% Delete one of the deployed RabbitMQ clusters & associated workloads 55 | rabbitmq-stream Deploy a RabbitMQ with a Stream Queue workload 56 | ssh-k3s SSH into the K3S host`` 57 | ``` 58 | -------------------------------------------------------------------------------- /s01/e07/k8s/prometheus-stack/traefik-servicemonitor.yml: -------------------------------------------------------------------------------- 1 | apiVersion: monitoring.coreos.com/v1 2 | kind: ServiceMonitor 3 | metadata: 4 | labels: 5 | app: traefik 6 | release: prometheus 7 | name: traefik 8 | spec: 9 | endpoints: 10 | - port: metrics 11 | namespaceSelector: 12 | matchNames: 13 | - kube-system 14 | selector: 15 | matchLabels: 16 | app: traefik 17 | 18 | -------------------------------------------------------------------------------- /s01/e07/k8s/rabbitmq/default-metrics.yml: -------------------------------------------------------------------------------- 1 | apiVersion: rabbitmq.com/v1beta1 2 | kind: RabbitmqCluster 3 | metadata: 4 | name: default-metrics 5 | # kubectl explain rmq.spec 6 | spec: 7 | # https://hub.docker.com/_/rabbitmq?tab=tags 8 | image: rabbitmq:3.8.9-management 9 | replicas: 1 10 | resources: 11 | limits: 12 | memory: 8G 13 | cpu: 4 14 | requests: 15 | memory: 8G 16 | cpu: 4 17 | rabbitmq: 18 | additionalConfig: | 19 | # https://www.rabbitmq.com/configure.html 20 | vm_memory_high_watermark.absolute = 7GB 21 | vm_memory_high_watermark_paging_ratio = 0.99 22 | 23 | # Statistics collection interval in milliseconds (default=5000) 24 | collect_statistics_interval = 5000 25 | 26 | # supported values: basic, detailed, none 27 | # default=basic 28 | management.rates_mode = basic 29 | 30 | # defaults: 5, 60, 1800 31 | management.sample_retention_policies.global.minute = 5 32 | management.sample_retention_policies.global.hour = 60 33 | management.sample_retention_policies.global.day = 1800 34 | 35 | # defaults: 5, 60 36 | management.sample_retention_policies.basic.minute = 5 37 | management.sample_retention_policies.basic.hour = 60 38 | 39 | # management metrics can be disabled altogether 40 | management_agent.disable_metrics_collector = false 41 | 42 | # default: 5 43 | management.sample_retention_policies.detailed.10 = 5 44 | --- 45 | apiVersion: apps/v1 46 | kind: Deployment 47 | metadata: 48 | name: default-metrics-rabbitmq-perf-test 49 | spec: 50 | replicas: 1 51 | selector: 52 | matchLabels: 53 | app: rabbitmq-perf-test 54 | template: 55 | metadata: 56 | labels: 57 | app: rabbitmq-perf-test 58 | annotations: 59 | prometheus.io/port: "8080" 60 | prometheus.io/scrape: "true" 61 | spec: 62 | containers: 63 | - name: perftest 64 | # https://hub.docker.com/r/pivotalrabbitmq/perf-test/tags 65 | image: pivotalrabbitmq/perf-test:2.12.0 66 | resources: 67 | limits: 68 | cpu: 2 69 | memory: 3G 70 | requests: 71 | cpu: 1 72 | memory: 1G 73 | command: 74 | - /bin/bash 75 | - -c 76 | args: 77 | - |- 78 | bin/runjava com.rabbitmq.perf.PerfTest \ 79 | --uri "amqp://$RABBITMQ_USER:$RABBITMQ_PASS@$RABBITMQ_SERVICE:5672/%2f" 80 | env: 81 | - name: RABBITMQ_USER 82 | valueFrom: 83 | secretKeyRef: 84 | name: default-metrics-rabbitmq-default-user 85 | key: username 86 | - name: RABBITMQ_PASS 87 | valueFrom: 88 | secretKeyRef: 89 | name: default-metrics-rabbitmq-default-user 90 | key: password 91 | - name: RABBITMQ_SERVICE 92 | value: default-metrics-rabbitmq-client 93 | 94 | - name: PRODUCER_SCHEDULER_THREADS 95 | value: "500" 96 | - name: PRODUCERS 97 | value: "5000" 98 | - name: PRODUCER_RANDOM_START_DELAY 99 | value: "600" 100 | - name: PUBLISHING_INTERVAL 101 | value: "10" 102 | - name: CONFIRM 103 | value: "1" 104 | - name: ROUTING_KEY 105 | value: "default-metrics-non-durable" 106 | - name: SIZE 107 | value: "100" 108 | 109 | - name: CONSUMERS_THREAD_POOLS 110 | value: "500" 111 | - name: CONSUMERS 112 | value: "5000" 113 | - name: CONSUMER_LATENCY 114 | value: "10000" 115 | 116 | - name: HEARTBEAT_SENDER_THREADS 117 | value: "500" 118 | - name: NIO_THREAD_POOL 119 | value: "500" 120 | 121 | - name: QUEUE_PATTERN 122 | value: "default-metrics-non-durable-%d" 123 | - name: QUEUE_PATTERN_FROM 124 | value: "1" 125 | - name: QUEUE_PATTERN_TO 126 | value: "5000" 127 | - name: QUEUE_ARGS 128 | value: "x-max-length=100" 129 | - name: AUTO_DELETE 130 | value: "false" 131 | 132 | - name: METRICS_PROMETHEUS 133 | value: "true" 134 | -------------------------------------------------------------------------------- /s01/e07/k8s/rabbitmq/minimal-metrics.yml: -------------------------------------------------------------------------------- 1 | apiVersion: rabbitmq.com/v1beta1 2 | kind: RabbitmqCluster 3 | metadata: 4 | name: minimal-metrics 5 | # kubectl explain rmq.spec 6 | spec: 7 | # https://hub.docker.com/_/rabbitmq?tab=tags 8 | image: rabbitmq:3.8.9-management 9 | replicas: 1 10 | resources: 11 | limits: 12 | memory: 8G 13 | cpu: 4 14 | requests: 15 | memory: 8G 16 | cpu: 4 17 | rabbitmq: 18 | additionalConfig: | 19 | # https://www.rabbitmq.com/configure.html 20 | vm_memory_high_watermark.absolute = 7GB 21 | vm_memory_high_watermark_paging_ratio = 0.99 22 | 23 | # Statistics collection interval in milliseconds (default=5000) 24 | collect_statistics_interval = 10000 25 | 26 | # supported values: basic, detailed, none 27 | # default=basic 28 | management.rates_mode = basic 29 | 30 | # defaults: 5, 60, 1800 31 | management.sample_retention_policies.global.minute = 10 32 | management.sample_retention_policies.global.hour = 300 33 | management.sample_retention_policies.global.day = 1800 34 | 35 | # defaults: 5, 60 36 | management.sample_retention_policies.basic.minute = 10 37 | management.sample_retention_policies.basic.hour = 300 38 | 39 | # management metrics can be disabled altogether 40 | management_agent.disable_metrics_collector = false 41 | 42 | # default: 5 43 | management.sample_retention_policies.detailed.10 = 60 44 | --- 45 | apiVersion: apps/v1 46 | kind: Deployment 47 | metadata: 48 | name: minimal-metrics-rabbitmq-perf-test 49 | spec: 50 | replicas: 1 51 | selector: 52 | matchLabels: 53 | app: rabbitmq-perf-test 54 | template: 55 | metadata: 56 | labels: 57 | app: rabbitmq-perf-test 58 | annotations: 59 | prometheus.io/port: "8080" 60 | prometheus.io/scrape: "true" 61 | spec: 62 | containers: 63 | - name: perftest 64 | # https://hub.docker.com/r/pivotalrabbitmq/perf-test/tags 65 | image: pivotalrabbitmq/perf-test:2.12.0 66 | resources: 67 | limits: 68 | cpu: 2 69 | memory: 3G 70 | requests: 71 | cpu: 1 72 | memory: 1G 73 | command: 74 | - /bin/bash 75 | - -c 76 | args: 77 | - |- 78 | bin/runjava com.rabbitmq.perf.PerfTest \ 79 | --uri "amqp://$RABBITMQ_USER:$RABBITMQ_PASS@$RABBITMQ_SERVICE:5672/%2f" 80 | env: 81 | - name: RABBITMQ_USER 82 | valueFrom: 83 | secretKeyRef: 84 | name: minimal-metrics-rabbitmq-default-user 85 | key: username 86 | - name: RABBITMQ_PASS 87 | valueFrom: 88 | secretKeyRef: 89 | name: minimal-metrics-rabbitmq-default-user 90 | key: password 91 | - name: RABBITMQ_SERVICE 92 | value: minimal-metrics-rabbitmq-client 93 | 94 | - name: PRODUCER_SCHEDULER_THREADS 95 | value: "500" 96 | - name: PRODUCERS 97 | value: "5000" 98 | - name: PRODUCER_RANDOM_START_DELAY 99 | value: "600" 100 | - name: PUBLISHING_INTERVAL 101 | value: "10" 102 | - name: CONFIRM 103 | value: "1" 104 | - name: ROUTING_KEY 105 | value: "minimal-metrics-non-durable" 106 | - name: SIZE 107 | value: "100" 108 | 109 | - name: CONSUMERS_THREAD_POOLS 110 | value: "500" 111 | - name: CONSUMERS 112 | value: "5000" 113 | - name: CONSUMER_LATENCY 114 | value: "10000" 115 | 116 | - name: HEARTBEAT_SENDER_THREADS 117 | value: "500" 118 | - name: NIO_THREAD_POOL 119 | value: "500" 120 | 121 | - name: QUEUE_PATTERN 122 | value: "minimal-metrics-non-durable-%d" 123 | - name: QUEUE_PATTERN_FROM 124 | value: "1" 125 | - name: QUEUE_PATTERN_TO 126 | value: "5000" 127 | - name: QUEUE_ARGS 128 | value: "x-max-length=100" 129 | - name: AUTO_DELETE 130 | value: "false" 131 | 132 | - name: METRICS_PROMETHEUS 133 | value: "true" 134 | -------------------------------------------------------------------------------- /s01/e07/k8s/rabbitmq/perftest.help: -------------------------------------------------------------------------------- 1 | ### http://github.com/rabbitmq/rabbitmq-perf-test 2 | 3 | ### docker run -it --rm pivotalrabbitmq/perf-test --help --env 4 | # 5 | For multi-value options, separate values with commas, e.g. VARIABLE_RATE='100:60,1000:10,500:15' 6 | AUTO_DELETE should the queue be auto-deleted, default 7 | is true 8 | AUTOACK auto ack 9 | BODY comma-separated list of files to use in 10 | message bodies 11 | BODY_CONTENT_TYPE body content-type 12 | BODY_COUNT number of pre-generated message bodies. 13 | Use with --json-body. Default is 100. 14 | BODY_FIELD_COUNT number of pre-generated fields and values 15 | for body. Use with --json-body. Default is 16 | 1000. 17 | CMESSAGES consumer message count 18 | CONFIRM max unconfirmed publishes 19 | CONFIRM_TIMEOUT waiting timeout for unconfirmed publishes 20 | before failing (in seconds) 21 | CONNECTION_RECOVERY_INTERVAL connection recovery interval in seconds. 22 | Default is 5 seconds. Interval syntax, 23 | e.g. 30-60, is supported to specify an 24 | random interval between 2 values between 25 | each attempt. 26 | CONSUMER_ARGS consumer arguments as key/values pairs, 27 | separated by commas, e.g. x-priority=10 28 | CONSUMER_CHANNEL_COUNT channels per consumer 29 | CONSUMER_LATENCY consumer latency in microseconds 30 | CONSUMER_RATE consumer rate limit 31 | CONSUMERS consumer count 32 | CONSUMERS_THREAD_POOLS number of thread pools to use for all 33 | consumers, default is to use a thread pool 34 | for each consumer 35 | CTXSIZE consumer tx size 36 | DISABLE_CONNECTION_RECOVERY disable automatic connection recovery 37 | EXCHANGE exchange name 38 | EXCLUSIVE use server-named exclusive queues. Such 39 | queues can only be used by their declaring 40 | connection! 41 | FLAG message flag(s), supported values: 42 | persistent and mandatory. Use the option 43 | several times to specify several values. 44 | FRAMEMAX frame max 45 | GLOBAL_QOS channel prefetch count 46 | HEARTBEAT heartbeat interval 47 | HEARTBEAT_SENDER_THREADS number of threads for producers and 48 | consumers heartbeat senders 49 | ID test ID 50 | INTERVAL sampling interval in seconds 51 | JSON_BODY generate a random JSON document for 52 | message body. Use with --size. 53 | LEGACY_METRICS display legacy metrics (min/avg/max 54 | latency) 55 | MESSAGE_PROPERTIES message properties as key/value pairs, 56 | separated by commas, e.g. priority=5 57 | METRICS_HELP show metrics usage 58 | MULTI_ACK_EVERY multi ack every 59 | NACK nack and requeue messages 60 | NIO_THREAD_POOL size of NIO thread pool, should be 61 | slightly higher than number of NIO threads 62 | NIO_THREADS number of NIO threads to use 63 | OUTPUT_FILE output file for timing results 64 | PMESSAGES producer message count 65 | POLLING use basic.get to consume messages. Do not 66 | use this in real applications. 67 | POLLING_INTERVAL time to wait before polling with 68 | basic.get, in millisecond, default is 0. 69 | PREDECLARED allow use of predeclared objects 70 | PRODUCER_CHANNEL_COUNT channels per producer 71 | PRODUCER_RANDOM_START_DELAY max random delay in seconds to start 72 | producers 73 | PRODUCER_SCHEDULER_THREADS number of threads to use when using 74 | --publishing-interval 75 | PRODUCERS producer count 76 | PTXSIZE producer tx size 77 | PUBLISHING_INTERVAL publishing interval in seconds (opposite 78 | of producer rate limit) 79 | QOS consumer prefetch count 80 | QUEUE queue name 81 | QUEUE_ARGS queue arguments as key/value pairs, 82 | separated by commas, e.g. x-max-length=10 83 | QUEUE_PATTERN queue name pattern for creating queues in 84 | sequence 85 | QUEUE_PATTERN_FROM queue name pattern range start (inclusive) 86 | QUEUE_PATTERN_TO queue name pattern range end (inclusive) 87 | RANDOM_ROUTING_KEY use random routing key per message 88 | RATE producer rate limit 89 | ROUTING_KEY routing key 90 | ROUTING_KEY_CACHE_SIZE size of the random routing keys cache. See 91 | --random-routing-key. 92 | SASL_EXTERNAL use SASL EXTERNAL authentication, default 93 | is false. Set to true if using client 94 | certificate authentication with the 95 | rabbitmq_auth_mechanism_ssl plugin. 96 | SERVERS_STARTUP_TIMEOUT start timeout in seconds (in case the 97 | servers(s) is (are) not available when the 98 | run starts). Default is to fail 99 | immediately if the servers(s) is (are) not 100 | available. 101 | SERVERS_UP_LIMIT number of available servers needed before 102 | starting the run. Used in conjunction with 103 | --servers-start-timeout. Default is 104 | deduced from --uri or --uris. 105 | SHUTDOWN_TIMEOUT shutdown timeout, default is 5 seconds 106 | SIZE message size in bytes 107 | SKIP_BINDING_QUEUES don't bind queues to the exchange 108 | SLOW_START start consumers slowly (1 sec delay 109 | between each) 110 | TIME run duration in seconds (unlimited by 111 | default) 112 | TYPE exchange type 113 | URI connection URI 114 | URIS connection URIs (separated by commas) 115 | USE_DEFAULT_SSL_CONTEXT use JVM default SSL context 116 | USE_MILLIS should latency be collected in 117 | milliseconds, default is false. Set to 118 | true if producers are consumers run on 119 | different machines. 120 | VARIABLE_LATENCY variable consumer processing latency with 121 | [MICROSECONDS]:[DURATION] syntax, where 122 | [MICROSECONDS] integer >= 0 and [DURATION] 123 | integer > 0. Use the option several times 124 | to specify several values. 125 | VARIABLE_RATE variable publishing rate with 126 | [RATE]:[DURATION] syntax, where [RATE] 127 | integer >= 0 and [DURATION] integer > 0. 128 | Use the option several times to specify 129 | several values. 130 | VARIABLE_SIZE variable message size with 131 | [SIZE]:[DURATION] syntax, where [SIZE] 132 | integer > 0 and [DURATION] integer > 0. 133 | Use the option several times to specify 134 | several values. 135 | 136 | ### docker run -it --rm pivotalrabbitmq/perf-test --metrics-help --env 137 | # 138 | METRICS_CLASS_LOADER enable JVM class loader metrics 139 | METRICS_CLIENT enable client metrics 140 | METRICS_DATADOG enable Datadog metrics 141 | METRICS_DATADOG_API_KEY Datadog API key 142 | METRICS_DATADOG_APPLICATION_KEY Datadog application key 143 | METRICS_DATADOG_DESCRIPTIONS if meter descriptions should be sent to 144 | Datadog 145 | METRICS_DATADOG_HOST_TAG tag that will be mapped to "host" when 146 | shipping metrics to datadog 147 | METRICS_DATADOG_STEP_SIZE step size (reporting frequency) to use 148 | in seconds, default is 10 seconds 149 | METRICS_DATADOG_URI URI to ship metrics, useful when using 150 | a proxy, default is 151 | https://app.datadoghq.com 152 | METRICS_JMX enable JMX metrics 153 | METRICS_JVM_GC enable JVM GC metrics 154 | METRICS_JVM_MEMORY enable JVM memory metrics 155 | METRICS_JVM_THREAD enable JVM thread metrics 156 | METRICS_PREFIX prefix for PerfTest metrics, default is 157 | perftest_ 158 | METRICS_PROCESSOR enable processor metrics (gathered by 159 | JVM) 160 | METRICS_PROMETHEUS enable Prometheus metrics 161 | METRICS_PROMETHEUS_ENDPOINT the HTTP metrics endpoint, default is 162 | /metrics 163 | METRICS_PROMETHEUS_PORT the port to launch the HTTP metrics 164 | endpoint on, default is 8080 165 | METRICS_TAGS metrics tags as key-value pairs 166 | separated by commas 167 | -------------------------------------------------------------------------------- /s01/e07/k8s/rabbitmq/prometheus-metrics.yml: -------------------------------------------------------------------------------- 1 | apiVersion: rabbitmq.com/v1beta1 2 | kind: RabbitmqCluster 3 | metadata: 4 | name: prometheus-metrics 5 | # kubectl explain rmq.spec 6 | spec: 7 | # https://hub.docker.com/_/rabbitmq?tab=tags 8 | image: rabbitmq:3.8.9-management 9 | replicas: 1 10 | resources: 11 | limits: 12 | memory: 8G 13 | cpu: 4 14 | requests: 15 | memory: 8G 16 | cpu: 4 17 | rabbitmq: 18 | additionalConfig: | 19 | # https://www.rabbitmq.com/configure.html 20 | vm_memory_high_watermark.absolute = 7GB 21 | vm_memory_high_watermark_paging_ratio = 0.99 22 | 23 | # management metrics can be disabled altogether 24 | management_agent.disable_metrics_collector = true 25 | --- 26 | apiVersion: apps/v1 27 | kind: Deployment 28 | metadata: 29 | name: prometheus-metrics-rabbitmq-perf-test 30 | spec: 31 | replicas: 1 32 | selector: 33 | matchLabels: 34 | app: rabbitmq-perf-test 35 | template: 36 | metadata: 37 | labels: 38 | app: rabbitmq-perf-test 39 | annotations: 40 | prometheus.io/port: "8080" 41 | prometheus.io/scrape: "true" 42 | spec: 43 | containers: 44 | - name: perftest 45 | # https://hub.docker.com/r/pivotalrabbitmq/perf-test/tags 46 | image: pivotalrabbitmq/perf-test:2.12.0 47 | resources: 48 | limits: 49 | cpu: 2 50 | memory: 3G 51 | requests: 52 | cpu: 1 53 | memory: 1G 54 | command: 55 | - /bin/bash 56 | - -c 57 | args: 58 | - |- 59 | bin/runjava com.rabbitmq.perf.PerfTest \ 60 | --uri "amqp://$RABBITMQ_USER:$RABBITMQ_PASS@$RABBITMQ_SERVICE:5672/%2f" 61 | env: 62 | - name: RABBITMQ_USER 63 | valueFrom: 64 | secretKeyRef: 65 | name: prometheus-metrics-rabbitmq-default-user 66 | key: username 67 | - name: RABBITMQ_PASS 68 | valueFrom: 69 | secretKeyRef: 70 | name: prometheus-metrics-rabbitmq-default-user 71 | key: password 72 | - name: RABBITMQ_SERVICE 73 | value: prometheus-metrics-rabbitmq-client 74 | 75 | - name: PRODUCER_SCHEDULER_THREADS 76 | value: "500" 77 | - name: PRODUCERS 78 | value: "5000" 79 | - name: PRODUCER_RANDOM_START_DELAY 80 | value: "600" 81 | - name: PUBLISHING_INTERVAL 82 | value: "10" 83 | - name: CONFIRM 84 | value: "1" 85 | - name: ROUTING_KEY 86 | value: "prometheus-metrics-non-durable" 87 | - name: SIZE 88 | value: "100" 89 | 90 | - name: CONSUMERS_THREAD_POOLS 91 | value: "500" 92 | - name: CONSUMERS 93 | value: "5000" 94 | - name: CONSUMER_LATENCY 95 | value: "10000" 96 | 97 | - name: HEARTBEAT_SENDER_THREADS 98 | value: "500" 99 | - name: NIO_THREAD_POOL 100 | value: "500" 101 | 102 | - name: QUEUE_PATTERN 103 | value: "prometheus-metrics-non-durable-%d" 104 | - name: QUEUE_PATTERN_FROM 105 | value: "1" 106 | - name: QUEUE_PATTERN_TO 107 | value: "5000" 108 | - name: QUEUE_ARGS 109 | value: "x-max-length=100" 110 | - name: AUTO_DELETE 111 | value: "false" 112 | 113 | - name: METRICS_PROMETHEUS 114 | value: "true" 115 | -------------------------------------------------------------------------------- /s01/e07/k8s/rabbitmq/quorum.yml: -------------------------------------------------------------------------------- 1 | apiVersion: rabbitmq.com/v1beta1 2 | kind: RabbitmqCluster 3 | metadata: 4 | name: quorum 5 | # kubectl explain rmq.spec 6 | spec: 7 | # https://hub.docker.com/_/rabbitmq?tab=tags 8 | image: rabbitmq:3.8.9-management 9 | replicas: 3 10 | resources: 11 | limits: 12 | memory: 8G 13 | cpu: 4 14 | requests: 15 | memory: 8G 16 | cpu: 4 17 | rabbitmq: 18 | additionalConfig: | 19 | # https://www.rabbitmq.com/configure.html 20 | vm_memory_high_watermark.absolute = 7G 21 | vm_memory_high_watermark_paging_ratio = 0.99 22 | 23 | # https://www.rabbitmq.com/prometheus.html#metric-aggregation 24 | prometheus.return_per_object_metrics = true 25 | --- 26 | apiVersion: apps/v1 27 | kind: Deployment 28 | metadata: 29 | name: quorum-rabbitmq-perf-test 30 | spec: 31 | replicas: 1 32 | selector: 33 | matchLabels: 34 | app: quorum-rabbitmq-perf-test 35 | template: 36 | metadata: 37 | labels: 38 | app: quorum-rabbitmq-perf-test 39 | annotations: 40 | prometheus.io/port: "8080" 41 | prometheus.io/scrape: "true" 42 | spec: 43 | containers: 44 | - name: perftest 45 | # https://hub.docker.com/r/pivotalrabbitmq/perf-test/tags 46 | image: pivotalrabbitmq/perf-test:2.12.0 47 | resources: 48 | limits: 49 | cpu: 2 50 | memory: 2G 51 | requests: 52 | cpu: 1 53 | memory: 1G 54 | command: 55 | - /bin/bash 56 | - -c 57 | args: 58 | - |- 59 | bin/runjava com.rabbitmq.perf.PerfTest \ 60 | --uri "amqp://$RABBITMQ_USER:$RABBITMQ_PASS@$RABBITMQ_SERVICE:5672/%2f" 61 | env: 62 | - name: RABBITMQ_USER 63 | valueFrom: 64 | secretKeyRef: 65 | name: quorum-rabbitmq-default-user 66 | key: username 67 | - name: RABBITMQ_PASS 68 | valueFrom: 69 | secretKeyRef: 70 | name: quorum-rabbitmq-default-user 71 | key: password 72 | - name: RABBITMQ_SERVICE 73 | value: quorum-rabbitmq-client 74 | 75 | - name: PRODUCERS 76 | value: "100" 77 | - name: VARIABLE_RATE 78 | value: "1:30,10:30,2:30,0:60" 79 | - name: CONFIRM 80 | value: "1" 81 | - name: JSON_BODY 82 | value: "true" 83 | - name: SIZE 84 | value: "1000" 85 | - name: FLAG 86 | value: "persistent" 87 | - name: ROUTING_KEY 88 | value: "quorum" 89 | 90 | - name: CONSUMERS 91 | value: "200" 92 | - name: CONSUMER_LATENCY 93 | value: "50000" 94 | 95 | - name: QUEUE_PATTERN 96 | value: "quorum-%d" 97 | - name: QUEUE_PATTERN_FROM 98 | value: "1" 99 | - name: QUEUE_PATTERN_TO 100 | value: "100" 101 | - name: QUEUE_ARGS 102 | value: "x-max-length=1000,x-queue-type=quorum,x-max-in-memory-bytes=10000000" 103 | - name: AUTO_DELETE 104 | value: "false" 105 | 106 | - name: METRICS_PROMETHEUS 107 | value: "true" 108 | -------------------------------------------------------------------------------- /s01/e07/k8s/rabbitmq/stream-perftest.help: -------------------------------------------------------------------------------- 1 | Tests the performance of stream queues in RabbitMQ. 2 | [COMMAND...] The COMMAND to display the usage help message for. 3 | Default: [] 4 | -a, --addresses=[,...] 5 | servers to connect to, e.g. localhost:5555, 6 | separated by commas 7 | Default: localhost:5555 8 | -bs, --batch-size= 9 | size of a batch of published messages 10 | Default: 100 11 | -c, --confirms= outstanding confirms 12 | Default: 10000 13 | -cc, --codec= 14 | class of codec to use. Aliases: qpid, simple. 15 | Default: qpid 16 | -ce, --commit-every= 17 | the frequency of offset commit 18 | Default: 0 19 | -cr, --credit= credit requested on acknowledgment 20 | Default: 1 21 | -h, --help Show usage help for the help command and exit. 22 | -icr, --initial-credit= 23 | initial credit when registering a consumer 24 | Default: 10 25 | -mlb, --max-length-bytes= 26 | max size of created streams 27 | Default: 20gb 28 | -mss, --max-segment-size= 29 | max size of segments 30 | Default: 500mb 31 | -o, --offset= offset to start listening from. Valid values are 32 | 'first', 'last', 'next', an unsigned long, or an 33 | ISO 8601 formatted timestamp (eg. 2020-06-03T07: 34 | 45:54Z). 35 | Default: first 36 | -p, --pre-declared whether streams are already declared or not 37 | -pw, --password= 38 | password to use for connecting 39 | Default: guest 40 | -r, --rate= maximum rate of published messages 41 | Default: -1 42 | -s, --size= size of messages in bytes 43 | Default: 10 44 | -ses, --sub-entry-size= 45 | number of messages packed into a normal message 46 | entry 47 | Default: 1 48 | -sf, --summary-file generate a summary file with metrics 49 | -st, --streams=[,...] 50 | stream(s) to send to and consume from, separated 51 | by commas 52 | Default: stream1 53 | -u, --username= username to use for connecting 54 | Default: guest 55 | -v, --version show version information 56 | -x, --producers= 57 | number of producers 58 | Default: 1 59 | -y, --consumers= 60 | number of consumers 61 | Default: 1 62 | -------------------------------------------------------------------------------- /s01/e07/k8s/rabbitmq/stream.yml: -------------------------------------------------------------------------------- 1 | apiVersion: rabbitmq.com/v1beta1 2 | kind: RabbitmqCluster 3 | metadata: 4 | name: stream 5 | # kubectl explain rmq.spec 6 | spec: 7 | # https://hub.docker.com/r/pivotalrabbitmq/rabbitmq-stream/tags 8 | image: pivotalrabbitmq/rabbitmq-stream 9 | replicas: 1 10 | resources: 11 | limits: 12 | memory: 2G 13 | cpu: 4 14 | requests: 15 | memory: 2G 16 | cpu: 4 17 | rabbitmq: 18 | additionalConfig: | 19 | # https://www.rabbitmq.com/configure.html 20 | vm_memory_high_watermark.absolute = 1GB 21 | additionalPlugins: 22 | - rabbitmq_stream 23 | override: 24 | clientService: 25 | spec: 26 | ports: 27 | - name: stream 28 | protocol: TCP 29 | port: 5555 30 | statefulSet: 31 | spec: 32 | template: 33 | spec: 34 | containers: 35 | - name: rabbitmq 36 | ports: 37 | - containerPort: 5555 38 | name: stream 39 | protocol: TCP 40 | --- 41 | apiVersion: apps/v1 42 | kind: Deployment 43 | metadata: 44 | name: stream-rabbitmq-perf-test 45 | spec: 46 | replicas: 1 47 | selector: 48 | matchLabels: 49 | app: stream-rabbitmq-perf-test 50 | template: 51 | metadata: 52 | labels: 53 | app: stream-rabbitmq-perf-test 54 | annotations: 55 | prometheus.io/port: "8080" 56 | prometheus.io/scrape: "true" 57 | spec: 58 | containers: 59 | - name: perftest 60 | # https://hub.docker.com/r/pivotalrabbitmq/stream-perf-test/tags 61 | image: pivotalrabbitmq/stream-perf-test 62 | resources: 63 | limits: 64 | cpu: 2 65 | memory: 2G 66 | requests: 67 | cpu: 1 68 | memory: 1G 69 | command: 70 | - /bin/bash 71 | - -c 72 | args: 73 | - |- 74 | java -Dio.netty.processId=1 -jar stream-perf-test.jar \ 75 | --addresses $RABBITMQ_SERVICE:5555 \ 76 | --username $RABBITMQ_USER \ 77 | --password $RABBITMQ_PASS \ 78 | --max-length-bytes=5000000000 \ 79 | --producers=1 \ 80 | --consumers=1 81 | env: 82 | - name: RABBITMQ_USER 83 | valueFrom: 84 | secretKeyRef: 85 | name: stream-rabbitmq-default-user 86 | key: username 87 | - name: RABBITMQ_PASS 88 | valueFrom: 89 | secretKeyRef: 90 | name: stream-rabbitmq-default-user 91 | key: password 92 | - name: RABBITMQ_SERVICE 93 | value: stream-rabbitmq-client 94 | -------------------------------------------------------------------------------- /s01/e07/video.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rabbitmq/tgir/87c753ae5bffa8cb8b321ca58297858153adeb6f/s01/e07/video.jpg -------------------------------------------------------------------------------- /s01/e08/.env: -------------------------------------------------------------------------------- 1 | ../../.env -------------------------------------------------------------------------------- /s01/e08/.gitignore: -------------------------------------------------------------------------------- 1 | .config 2 | .envrc 3 | bin 4 | tmp 5 | .DS_Store 6 | -------------------------------------------------------------------------------- /s01/e08/Makefile: -------------------------------------------------------------------------------- 1 | include $(dir $(word $(words $(MAKEFILE_LIST)),$(MAKEFILE_LIST)))/../../Makefile 2 | 3 | TGIR := tgir-s01e08 4 | DKE_CLUSTER := $(TGIR) 5 | DKE_NODES_COUNT ?= 3 6 | # make dke-regions 7 | # Because Frankfurt is closer to @GSantomaggio than either London or Amsterdam 8 | DKE_REGION ?= fra1 9 | # make dke-sizes 10 | DKE_NODES_TYPE ?= s-2vcpu-4gb 11 | # make dke-versions 12 | DKE_VERSION ?= 1.19.3-do.2 13 | # Keeping this in sync with DKE version 14 | KUBECTL_VERSION = 1.19.3 15 | 16 | KUBECONFIG_DIR := $(XDG_CONFIG_HOME)/kubectl 17 | KUBECONFIG := $(KUBECONFIG_DIR)/config 18 | export KUBECONFIG 19 | 20 | $(KUBECONFIG_DIR): 21 | @mkdir -p $(@) 22 | 23 | .PHONY: k9s 24 | k9s: | $(K9S) $(KUBECONFIG) ## Interact with our K8S cluster via a terminal UI 25 | $(K9S) --all-namespaces 26 | 27 | define ENV 28 | export PATH=$(LOCAL_BIN):$$PATH 29 | export XDG_CONFIG_HOME="$(XDG_CONFIG_HOME)" 30 | export KUBECONFIG="$(KUBECONFIG)" 31 | unalias k 2>/dev/null; alias k=kubectl 32 | unalias m 2>/dev/null; alias m=make 33 | endef 34 | export ENV 35 | .PHONY: env 36 | env:: | $(KUBECONFIG_DIR) ## Configure shell env - eval "$(make env)" OR source .env 37 | @echo "$$ENV" 38 | 39 | DOCTL_RELEASES := https://github.com/digitalocean/doctl/releases 40 | DOCTL_VERSION := 1.54.0 41 | DOCTL_BIN_DIR := doctl-$(DOCTL_VERSION)-$(platform)-amd64 42 | DOCTL_URL := https://github.com/digitalocean/doctl/releases/download/v$(DOCTL_VERSION)/doctl-$(DOCTL_VERSION)-$(platform)-amd64.tar.gz 43 | DOCTL := $(LOCAL_BIN)/$(DOCTL_BIN_DIR)/doctl 44 | $(DOCTL): | $(CURL) $(LOCAL_BIN) 45 | $(CURL) --progress-bar --fail --location --output $(LOCAL_BIN)/$(DOCTL_BIN_DIR).tar.gz "$(DOCTL_URL)" 46 | mkdir -p $(LOCAL_BIN)/$(DOCTL_BIN_DIR) && tar zxf $(LOCAL_BIN)/$(DOCTL_BIN_DIR).tar.gz -C $(LOCAL_BIN)/$(DOCTL_BIN_DIR) 47 | touch $(DOCTL) 48 | chmod +x $(DOCTL) 49 | $(DOCTL) version | grep $(DOCTL_VERSION) 50 | ln -sf $(DOCTL) $(LOCAL_BIN)/doctl 51 | .PHONY: doctl 52 | doctl: $(DOCTL) 53 | 54 | DOCTL_CONFIG_DIR := $(XDG_CONFIG_HOME)/doctl 55 | $(DOCTL_CONFIG_DIR): 56 | mkdir -p $(@) 57 | DOCTL_CONFIG := $(DOCTL_CONFIG_DIR)/config.yaml 58 | DOCTL_WITH_CONFIG = $(DOCTL) --config $(DOCTL_CONFIG) 59 | $(DOCTL_CONFIG): | $(DOCTL_CONFIG_DIR) $(DOCTL) 60 | $(DOCTL_WITH_CONFIG) auth init 61 | 62 | .PHONY: all 63 | all:: ## Setup all resources 64 | 65 | .PHONY: clean 66 | clean:: ## Remove all installed resources 67 | 68 | 69 | .PHONY: dke 70 | dke: | $(DOCTL_CONFIG) $(KUBECONFIG_DIR) ## Create Digital Ocean Kubernetes (DKE) cluster 71 | $(DOCTL_WITH_CONFIG) kubernetes cluster get $(DKE_CLUSTER) \ 72 | || time $(DOCTL_WITH_CONFIG) kubernetes cluster create $(DKE_CLUSTER) \ 73 | --region $(DKE_REGION) \ 74 | --size $(DKE_NODES_TYPE) \ 75 | --version $(DKE_VERSION) \ 76 | --count $(DKE_NODES_COUNT) \ 77 | --auto-upgrade \ 78 | --maintenance-window saturday=20:00 79 | 80 | $(KUBECONFIG): | $(KUBECONFIG_DIR) $(KUBECTL) $(DOCTL) 81 | $(DOCTL_WITH_CONFIG) kubernetes cluster kubeconfig save $(DKE_CLUSTER) 82 | .PHONY: kubeconfig 83 | kubeconfig: $(KUBECONFIG) 84 | 85 | .PHONY: dke-ls 86 | dke-ls: | $(DOCTL_CONFIG) ## List DKE clusters 87 | $(DOCTL_WITH_CONFIG) kubernetes cluster list 88 | 89 | .PHONY: dke-rm 90 | dke-rm: | $(DOCTL_CONFIG) ## Delete DKE cluster 91 | $(DOCTL_WITH_CONFIG) kubernetes cluster delete $(DKE_CLUSTER) 92 | 93 | .PHONY: dke-regions 94 | dke-regions: | $(DOCTL_CONFIG) ## Show all regions where DKE can be deployed 95 | $(DOCTL_WITH_CONFIG) kubernetes options regions 96 | 97 | .PHONY: dke-sizes 98 | dke-sizes: | $(DOCTL_CONFIG) ## Show all size options for DKE nodes 99 | $(DOCTL_WITH_CONFIG) kubernetes options sizes 100 | 101 | .PHONY: dke-versions 102 | dke-versions: | $(DOCTL_CONFIG) ## Show all size options for DKE nodes 103 | $(DOCTL_WITH_CONFIG) kubernetes options versions 104 | 105 | # helm search repo --versions traefik/traefik 106 | TRAEFIK_VERSION ?= 9.11.0 107 | .PHONY: traefik 108 | traefik: | $(HELM) $(KUBECONFIG) ## 5. Install Traefik 109 | $(HELM) repo add traefik https://helm.traefik.io/traefik 110 | $(HELM) upgrade traefik traefik/traefik \ 111 | --install \ 112 | --version $(TRAEFIK_VERSION) \ 113 | --values yaml/values.yaml 114 | all:: traefik 115 | 116 | .PHONY: traefik-rm 117 | traefik-rm: | $(HELM) kubeconfig ## Remove Traefik 118 | $(HELM) uninstall traefik \ 119 | ; true 120 | clean:: traefik-rm 121 | 122 | # https://github.com/jetstack/cert-manager/releases 123 | CERT_MANAGER_VERSION ?= 1.1.0 124 | CERT_MANAGER_MANIFEST = https://github.com/jetstack/cert-manager/releases/download/v$(CERT_MANAGER_VERSION)/cert-manager.yaml 125 | CERT_MANAGER_NAMESPACE = cert-manager 126 | .PHONY: cert-manager 127 | cert-manager: | kubeconfig ## 3. Deploy cert-manager for Let's Encrypt TLS certs 128 | $(KUBECTL) apply \ 129 | --filename $(CERT_MANAGER_MANIFEST) 130 | all:: cert-manager 131 | 132 | .PHONY: cert-manager-rm 133 | cert-manager-rm: | kubeconfig 134 | $(KUBECTL) delete \ 135 | --filename $(CERT_MANAGER_MANIFEST) \ 136 | --ignore-not-found 137 | clean:: cert-manager-rm 138 | 139 | .PHONY: cert-manager-cloudflare 140 | cert-manager-cloudflare: | cloudflare-secret kubeconfig ## 4. Enable rabbitmq.com TLS certs verification 141 | $(KUBECTL) apply \ 142 | --filename yaml/cert-manager-cloudflare-issuer.yaml 143 | all:: cert-manager-cloudflare 144 | 145 | .PHONY: cert-manager-cloudflare-rm 146 | cert-manager-cloudflare-rm: | cloudflare-secret-rm kubeconfig 147 | $(KUBECTL) delete \ 148 | --filename yaml/cert-manager-cloudflare-issuer.yaml \ 149 | --ignore-not-found 150 | $(KUBECTL) delete \ 151 | secret/letsencrypt-issuer-account-key \ 152 | --namespace $(CERT_MANAGER_NAMESPACE) \ 153 | --ignore-not-found 154 | clean:: cert-manager-cloudflare-rm 155 | 156 | CLOUDFLARE_GLOBAL_API_KEY_PATH = tmp/tgir-s01e08-cert-manager-cloudflare-global-api-key 157 | CLOUDFLARE_GLOBAL_API_KEY = "$$(cat $(CLOUDFLARE_GLOBAL_API_KEY_PATH))" 158 | .PHONY: cloudflare-secret 159 | cloudflare-secret: | kubeconfig 160 | ifeq ($(wildcard $(CLOUDFLARE_GLOBAL_API_KEY_PATH)),) 161 | @printf "Create a new Clouflare Global API key for cert-manager $(BOLD)https://dash.cloudflare.com/profile/api-tokens$(NORMAL)\n" 162 | @printf "And then write it to $(BOLD)$(CLOUDFLARE_GLOBAL_API_KEY_PATH)$(NORMAL)\n" 163 | @exit 1 164 | endif 165 | @$(KUBECTL) get secret cloudflare --namespace $(CERT_MANAGER_NAMESPACE) \ 166 | || ( \ 167 | $(KUBECTL) --dry-run=client --output=yaml \ 168 | create secret generic cloudflare \ 169 | --from-literal=global_api_key=$(CLOUDFLARE_GLOBAL_API_KEY) \ 170 | | $(KUBECTL) apply \ 171 | --namespace $(CERT_MANAGER_NAMESPACE) \ 172 | --filename - \ 173 | ) 174 | 175 | .PHONY: cloudflare-secret-rm 176 | cloudflare-secret-rm: | kubeconfig 177 | $(KUBECTL) delete \ 178 | secret/cloudflare \ 179 | --namespace $(CERT_MANAGER_NAMESPACE) \ 180 | --ignore-not-found 181 | 182 | # https://github.com/rabbitmq/cluster-operator/releases 183 | CLUSTER_OPERATOR_VERSION ?= 1.3.0 184 | CLUSTER_OPERATOR_MANIFEST = https://github.com/rabbitmq/cluster-operator/releases/download/v$(CLUSTER_OPERATOR_VERSION)/cluster-operator.yml 185 | .PHONY: cluster-operator 186 | cluster-operator: | kubeconfig ## 1. Install RabbitMQ Cluster Operator for easy & correct RabbitMQ clusters on K8S 187 | $(KUBECTL) apply --filename $(CLUSTER_OPERATOR_MANIFEST) 188 | all:: cluster-operator 189 | 190 | .PHONY: rabbitmq-%-insecure-rm 191 | rabbitmq-%-insecure-rm: | kubeconfig ## Delete RabbitMQ cluster & public IP 192 | kubectl delete --filename yaml/rabbitmq-$(*)-insecure.yaml --ignore-not-found 193 | clean:: rabbitmq-bugs-insecure-rm 194 | clean:: rabbitmq-bunny-insecure-rm 195 | 196 | .PHONY: rabbitmq-%-insecure 197 | rabbitmq-%-insecure: | kubeconfig ## 2. Deploy RabbitMQ cluster with a public IP 198 | kubectl apply --filename yaml/rabbitmq-$(*)-insecure.yaml 199 | 200 | .PHONY: rabbitmq-%-secure-rm 201 | rabbitmq-%-secure-rm: | kubeconfig ## Delete RabbitMQ cluster 202 | $(KUBECTL) delete --filename yaml/rabbitmq-$(*)-secure.yaml --ignore-not-found 203 | $(KUBECTL) delete --filename yaml/rabbitmq-$(*)-cert.yaml --ignore-not-found 204 | $(KUBECTL) delete secret/rabbitmq-$(*)-tls --ignore-not-found 205 | clean:: rabbitmq-bugs-secure-rm 206 | clean:: rabbitmq-bunny-secure-rm 207 | 208 | .PHONY: cluster-operator-rm 209 | cluster-operator-rm: 210 | $(KUBECTL) delete --filename $(CLUSTER_OPERATOR_MANIFEST) --ignore-not-found 211 | clean:: cluster-operator-rm 212 | 213 | .PHONY: rabbitmq-%-secure-perftest-rm 214 | rabbitmq-%-secure-perftest-rm: | kubeconfig ## Delete PerfTest that runs against RabbitMQ cluster secure public IP 215 | kubectl delete --filename yaml/rabbitmq-$(*)-secure-perftest.yaml --ignore-not-found 216 | clean:: rabbitmq-bugs-secure-perftest-rm 217 | clean:: rabbitmq-bunny-secure-perftest-rm 218 | 219 | .PHONY: rabbitmq-%-secure-perftest 220 | rabbitmq-%-secure-perftest: | kubeconfig ## 8. Run PerfTest against RabbitMQ cluster secure public IP 221 | kubectl apply --filename yaml/rabbitmq-$(*)-secure-perftest.yaml 222 | all:: rabbitmq-bugs-secure-perftest 223 | all:: rabbitmq-bunny-secure-perftest 224 | 225 | .PHONY: rabbitmq-%-secure 226 | rabbitmq-%-secure: | kubeconfig ## 6. Deploy a RabbitMQ cluster with secure public IP 227 | $(KUBECTL) apply --filename yaml/rabbitmq-$(*)-cert.yaml 228 | $(KUBECTL) apply --filename yaml/rabbitmq-$(*)-secure.yaml 229 | all:: rabbitmq-bugs-secure 230 | all:: rabbitmq-bunny-secure 231 | 232 | .PHONY: traefik-%-ingress-rm 233 | traefik-%-ingress-rm: | kubeconfig ## Deploy HTTPS and AMQPS Traefik ingress 234 | $(KUBECTL) delete --filename yaml/rabbitmq-$(*)-traefik-ingress-amqps.yaml --ignore-not-found 235 | $(KUBECTL) delete --filename yaml/rabbitmq-$(*)-traefik-ingress-https.yaml --ignore-not-found 236 | clean:: traefik-bugs-ingress-rm 237 | clean:: traefik-bunny-ingress-rm 238 | 239 | .PHONY: traefik-%-ingress 240 | traefik-%-ingress: | kubeconfig ## 7. Deploy HTTPS and AMQPS Traefik ingress 241 | $(KUBECTL) apply --filename yaml/rabbitmq-$(*)-traefik-ingress-amqps.yaml 242 | $(KUBECTL) apply --filename yaml/rabbitmq-$(*)-traefik-ingress-https.yaml 243 | all:: traefik-bugs-ingress 244 | all:: traefik-bunny-ingress 245 | -------------------------------------------------------------------------------- /s01/e08/README.md: -------------------------------------------------------------------------------- 1 | # TGIR S01E08: Secure public RabbitMQ clusters 2 | 3 | * Proposed via [rabbitmq/tgir#16](https://github.com/rabbitmq/tgir/issues/16) 4 | * Hosted by [@gerhardlazu](https://twitter.com/gerhardlazu) & [@GSantomaggio](https://twitter.com/GSantomaggio) 5 | * Published on: 2020-12-31 6 | * Video: https://www.youtube.com/watch?v=mPuMBB9_MRI 7 | 8 | 9 | 10 | How do you make your RabbitMQ clusters public & secure? Think HTTPS & AMQPS. 11 | 12 | What about using a single IP for multiple RabbitMQ clusters? 13 | 14 | And what about connection throttling? 15 | 16 | Gerhard & Gabriele try out RabbitMQ Cluster Operator v1.3, cert-manager v1.1 & Traefik v2.3 on Digital Ocean Kubernetes v1.19.3. 17 | 18 | Topics covered: 19 | - cert-manager with CloudFlare DNS 20 | - SNI in RabbitMQ, PerfTest & Erlang 21 | - Istio as an alternative to Traefik 22 | 23 | 24 | 25 | ## LINKS 26 | 27 | * [cert-manager](https://cert-manager.io/docs/) 28 | * [Traefik](https://doc.traefik.io/traefik/) 29 | * [cluster-operator - disableNonTLSListeners](https://github.com/rabbitmq/cluster-operator/pull/477) 30 | * [cluster-operator - Prometheus TLS](https://github.com/rabbitmq/cluster-operator/pull/533) 31 | * [cluster-operator - 1.2.0 to 1.3.0 rolling StatefulSet update?](https://github.com/rabbitmq/rabbitmq-server/discussions/2699) 32 | * [RabbitMQ PerfTest - -sni option](https://github.com/rabbitmq/rabbitmq-perf-test/pull/253) 33 | * [What is SNI?](https://www.cloudflare.com/learning/ssl/what-is-sni/) 34 | * [RabbitMQ - Using TLS in the Java Client](https://www.rabbitmq.com/ssl.html#java-client) 35 | * [Handle two RabbitMQ clusters with ISTIO](https://github.com/Gsantomaggio/k8s/tree/wip/rabbitmq_traffic) 36 | 37 | 38 | 39 | ## MAKE TARGETS 40 | 41 | ``` 42 | all Setup all resources 43 | cert-manager 3. Deploy cert-manager for Let's Encrypt TLS certs 44 | cert-manager-cloudflare 4. Enable rabbitmq.com TLS certs verification 45 | clean Remove all installed resources 46 | cluster-operator 1. Install RabbitMQ Cluster Operator for easy & correct RabbitMQ clusters on K8S 47 | dke Create Digital Ocean Kubernetes (DKE) cluster 48 | dke-ls List DKE clusters 49 | dke-regions Show all regions where DKE can be deployed 50 | dke-rm Delete DKE cluster 51 | dke-sizes Show all size options for DKE nodes 52 | dke-versions Show all size options for DKE nodes 53 | env Configure shell env - eval "$(make env)" OR source .env 54 | k9s Interact with our K8S cluster via a terminal UI 55 | rabbitmq-%-insecure 2. Deploy RabbitMQ cluster with a public IP 56 | rabbitmq-%-insecure-rm Delete RabbitMQ cluster & public IP 57 | rabbitmq-%-secure 6. Deploy a RabbitMQ cluster with secure public IP 58 | rabbitmq-%-secure-perftest 8. Run PerfTest against RabbitMQ cluster secure public IP 59 | rabbitmq-%-secure-perftest-rm Delete PerfTest that runs against RabbitMQ cluster secure public IP 60 | rabbitmq-%-secure-rm Delete RabbitMQ cluster 61 | traefik 5. Install Traefik 62 | traefik-%-ingress 7. Deploy HTTPS and AMQPS Traefik ingress 63 | traefik-%-ingress-rm Deploy HTTPS and AMQPS Traefik ingress 64 | traefik-rm Remove Traefik 65 | ``` 66 | -------------------------------------------------------------------------------- /s01/e08/video.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rabbitmq/tgir/87c753ae5bffa8cb8b321ca58297858153adeb6f/s01/e08/video.jpg -------------------------------------------------------------------------------- /s01/e08/yaml/cert-manager-cloudflare-issuer.yaml: -------------------------------------------------------------------------------- 1 | # https://cert-manager.io/docs/configuration/acme/ 2 | apiVersion: cert-manager.io/v1 3 | kind: ClusterIssuer 4 | metadata: 5 | name: letsencrypt 6 | namespace: cert-manager 7 | spec: 8 | acme: 9 | server: https://acme-v02.api.letsencrypt.org/directory 10 | preferredChain: "ISRG Root X1" 11 | privateKeySecretRef: 12 | name: letsencrypt-issuer-account-key 13 | solvers: 14 | - dns01: 15 | cloudflare: 16 | email: glazu@pivotal.io 17 | apiKeySecretRef: 18 | name: cloudflare 19 | key: global_api_key 20 | selector: 21 | dnsNames: 22 | - 'tgir-s01e08-bugs.rabbitmq.com' 23 | - 'tgir-s01e08-bunny.rabbitmq.com' 24 | -------------------------------------------------------------------------------- /s01/e08/yaml/rabbitmq-bugs-cert.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: cert-manager.io/v1 2 | kind: Certificate 3 | metadata: 4 | name: rabbitmq-bugs-cert 5 | spec: 6 | secretName: rabbitmq-bugs-tls 7 | dnsNames: 8 | - tgir-s01e08-bugs.rabbitmq.com 9 | issuerRef: 10 | name: letsencrypt 11 | kind: ClusterIssuer 12 | group: cert-manager.io 13 | -------------------------------------------------------------------------------- /s01/e08/yaml/rabbitmq-bugs-insecure.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rabbitmq.com/v1beta1 2 | kind: RabbitmqCluster 3 | metadata: 4 | name: rabbitmq-bugs 5 | spec: 6 | replicas: 1 7 | service: 8 | type: LoadBalancer 9 | -------------------------------------------------------------------------------- /s01/e08/yaml/rabbitmq-bugs-secure-perftest.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: rabbitmq-bugs-perf-test 5 | spec: 6 | replicas: 1 7 | selector: 8 | matchLabels: 9 | app: rabbitmq-bugs-perf-test 10 | template: 11 | metadata: 12 | labels: 13 | app: rabbitmq-bugs-perf-test 14 | annotations: 15 | prometheus.io/port: "8080" 16 | prometheus.io/scrape: "true" 17 | spec: 18 | containers: 19 | - name: perftest 20 | # https://hub.docker.com/r/pivotalrabbitmq/perf-test/tags 21 | # SNI support was added in 2.13.0: https://github.com/rabbitmq/rabbitmq-perf-test/pull/253 22 | image: pivotalrabbitmq/perf-test:2.13.0 23 | resources: 24 | limits: 25 | cpu: 0.5 26 | memory: 128Mi 27 | requests: 28 | cpu: 0.5 29 | memory: 128Mi 30 | command: 31 | - /bin/bash 32 | - -c 33 | args: 34 | - |- 35 | bin/runjava com.rabbitmq.perf.PerfTest \ 36 | --uri "amqps://$RABBITMQ_USER:$RABBITMQ_PASS@$RABBITMQ_PUBLIC_FQDN:5671/%2f" \ 37 | -sni $RABBITMQ_PUBLIC_FQDN 38 | env: 39 | - name: RABBITMQ_USER 40 | valueFrom: 41 | secretKeyRef: 42 | name: rabbitmq-bugs-default-user 43 | key: username 44 | - name: RABBITMQ_PASS 45 | valueFrom: 46 | secretKeyRef: 47 | name: rabbitmq-bugs-default-user 48 | key: password 49 | 50 | - name: RABBITMQ_PUBLIC_FQDN 51 | value: "tgir-s01e08-bugs.rabbitmq.com" 52 | - name: PRODUCERS 53 | value: "1" 54 | - name: VARIABLE_RATE 55 | value: "1:30,10:30,2:30,0:60" 56 | - name: CONFIRM 57 | value: "1" 58 | - name: JSON_BODY 59 | value: "true" 60 | - name: SIZE 61 | value: "1000" 62 | - name: FLAG 63 | value: "persistent" 64 | - name: ROUTING_KEY 65 | value: "quorum" 66 | 67 | - name: CONSUMERS 68 | value: "2" 69 | - name: CONSUMER_LATENCY 70 | value: "50000" 71 | 72 | - name: QUEUE_PATTERN 73 | value: "quorum-%d" 74 | - name: QUEUE_PATTERN_FROM 75 | value: "1" 76 | - name: QUEUE_PATTERN_TO 77 | value: "1" 78 | - name: QUEUE_ARGS 79 | value: "x-max-length=1000,x-queue-type=quorum,x-max-in-memory-bytes=1000000" 80 | - name: AUTO_DELETE 81 | value: "false" 82 | 83 | - name: METRICS_PROMETHEUS 84 | value: "true" 85 | -------------------------------------------------------------------------------- /s01/e08/yaml/rabbitmq-bugs-secure.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rabbitmq.com/v1beta1 2 | kind: RabbitmqCluster 3 | metadata: 4 | name: rabbitmq-bugs 5 | spec: 6 | replicas: 1 7 | tls: 8 | secretName: rabbitmq-bugs-tls 9 | disableNonTLSListeners: true 10 | -------------------------------------------------------------------------------- /s01/e08/yaml/rabbitmq-bugs-traefik-ingress-amqps.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: traefik.containo.us/v1alpha1 2 | kind: IngressRouteTCP 3 | metadata: 4 | name: rabbitmq-bugs-ingress 5 | spec: 6 | entryPoints: 7 | - amqps 8 | routes: 9 | - match: HostSNI(`tgir-s01e08-bugs.rabbitmq.com`) 10 | services: 11 | - name: rabbitmq-bugs 12 | port: 5671 13 | # TODO: why don't we see anything in Traefik logs when this is disabled? 14 | tls: 15 | passthrough: true 16 | -------------------------------------------------------------------------------- /s01/e08/yaml/rabbitmq-bugs-traefik-ingress-https.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: traefik.containo.us/v1alpha1 2 | kind: IngressRoute 3 | metadata: 4 | name: ingress-bugs-tls 5 | spec: 6 | entryPoints: 7 | - http-mgm-tls 8 | routes: 9 | - match: Host(`tgir-s01e08-bugs.rabbitmq.com`) 10 | kind: Rule 11 | services: 12 | - name: rabbitmq-bugs 13 | scheme: https 14 | port: 15671 15 | tls: 16 | secretName: rabbitmq-bugs-tls 17 | -------------------------------------------------------------------------------- /s01/e08/yaml/rabbitmq-bunny-cert.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: cert-manager.io/v1 2 | kind: Certificate 3 | metadata: 4 | name: rabbitmq-bunny-cert 5 | spec: 6 | secretName: rabbitmq-bunny-tls 7 | dnsNames: 8 | - tgir-s01e08-bunny.rabbitmq.com 9 | issuerRef: 10 | name: letsencrypt 11 | kind: ClusterIssuer 12 | group: cert-manager.io 13 | -------------------------------------------------------------------------------- /s01/e08/yaml/rabbitmq-bunny-insecure.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rabbitmq.com/v1beta1 2 | kind: RabbitmqCluster 3 | metadata: 4 | name: rabbitmq-bunny 5 | spec: 6 | replicas: 1 7 | service: 8 | type: LoadBalancer 9 | -------------------------------------------------------------------------------- /s01/e08/yaml/rabbitmq-bunny-secure-perftest.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: rabbitmq-bunny-perf-test 5 | spec: 6 | replicas: 1 7 | selector: 8 | matchLabels: 9 | app: rabbitmq-bunny-perf-test 10 | template: 11 | metadata: 12 | labels: 13 | app: rabbitmq-bunny-perf-test 14 | annotations: 15 | prometheus.io/port: "8080" 16 | prometheus.io/scrape: "true" 17 | spec: 18 | containers: 19 | - name: perftest 20 | # https://hub.docker.com/r/pivotalrabbitmq/perf-test/tags 21 | # SNI support was added in 2.13.0: https://github.com/rabbitmq/rabbitmq-perf-test/pull/253 22 | image: pivotalrabbitmq/perf-test:2.13.0 23 | resources: 24 | limits: 25 | cpu: 0.5 26 | memory: 128Mi 27 | requests: 28 | cpu: 0.5 29 | memory: 128Mi 30 | command: 31 | - /bin/bash 32 | - -c 33 | args: 34 | - |- 35 | bin/runjava com.rabbitmq.perf.PerfTest \ 36 | --uri "amqps://$RABBITMQ_USER:$RABBITMQ_PASS@$RABBITMQ_PUBLIC_FQDN:5671/%2f" \ 37 | -sni $RABBITMQ_PUBLIC_FQDN 38 | env: 39 | - name: RABBITMQ_USER 40 | valueFrom: 41 | secretKeyRef: 42 | name: rabbitmq-bunny-default-user 43 | key: username 44 | - name: RABBITMQ_PASS 45 | valueFrom: 46 | secretKeyRef: 47 | name: rabbitmq-bunny-default-user 48 | key: password 49 | 50 | - name: RABBITMQ_PUBLIC_FQDN 51 | value: "tgir-s01e08-bunny.rabbitmq.com" 52 | - name: PRODUCERS 53 | value: "1" 54 | - name: VARIABLE_RATE 55 | value: "1:30,10:30,2:30,0:60" 56 | - name: CONFIRM 57 | value: "1" 58 | - name: JSON_BODY 59 | value: "true" 60 | - name: SIZE 61 | value: "1000" 62 | - name: FLAG 63 | value: "persistent" 64 | - name: ROUTING_KEY 65 | value: "quorum" 66 | 67 | - name: CONSUMERS 68 | value: "2" 69 | - name: CONSUMER_LATENCY 70 | value: "50000" 71 | 72 | - name: QUEUE_PATTERN 73 | value: "quorum-%d" 74 | - name: QUEUE_PATTERN_FROM 75 | value: "1" 76 | - name: QUEUE_PATTERN_TO 77 | value: "1" 78 | - name: QUEUE_ARGS 79 | value: "x-max-length=1000,x-queue-type=quorum,x-max-in-memory-bytes=1000000" 80 | - name: AUTO_DELETE 81 | value: "false" 82 | 83 | - name: METRICS_PROMETHEUS 84 | value: "true" 85 | -------------------------------------------------------------------------------- /s01/e08/yaml/rabbitmq-bunny-secure.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rabbitmq.com/v1beta1 2 | kind: RabbitmqCluster 3 | metadata: 4 | name: rabbitmq-bunny 5 | spec: 6 | replicas: 1 7 | tls: 8 | secretName: rabbitmq-bunny-tls 9 | disableNonTLSListeners: true 10 | 11 | -------------------------------------------------------------------------------- /s01/e08/yaml/rabbitmq-bunny-traefik-ingress-amqps.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: traefik.containo.us/v1alpha1 2 | kind: IngressRouteTCP 3 | metadata: 4 | name: rabbitmq-bunny-ingress 5 | spec: 6 | entryPoints: 7 | - amqps 8 | routes: 9 | - match: HostSNI(`tgir-s01e08-bunny.rabbitmq.com`) 10 | services: 11 | - name: rabbitmq-bunny 12 | port: 5671 13 | # TODO: why don't we see anything in Traefik logs when this is disabled? 14 | tls: 15 | passthrough: true 16 | -------------------------------------------------------------------------------- /s01/e08/yaml/rabbitmq-bunny-traefik-ingress-https.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: traefik.containo.us/v1alpha1 2 | kind: IngressRoute 3 | metadata: 4 | name: ingress-bunny-tls 5 | spec: 6 | entryPoints: 7 | - http-mgm-tls 8 | routes: 9 | - match: Host(`tgir-s01e08-bunny.rabbitmq.com`) 10 | kind: Rule 11 | services: 12 | - name: rabbitmq-bunny 13 | scheme: https 14 | port: 15671 15 | tls: 16 | secretName: rabbitmq-bunny-tls 17 | -------------------------------------------------------------------------------- /s01/e08/yaml/steps: -------------------------------------------------------------------------------- 1 | * GL: Introduce the TGIR 2 | 3 | * GS: Start with sildes, what are the problems we need to solve? 4 | Expose RabbitMQ using kubernetes. 5 | We have different ways to do that. 6 | Let's start with the simplest way, k8s Operator using load-balancer option. 7 | We reached the aim, but... 8 | * GS: We are not happy, too many IP adress, clients are connected directly to 9 | the server. If you need to create another cluster and move the connections you need to change the IP, or if you want to implement a canary update it is not possible or hard to do. It is not a best prectise in k8s. 10 | * GS: Back to the slides and show the next steps. 11 | * GS: We need an ingress, what is an Ingress? (Link) 12 | Speak about traefik. 13 | We need it in TLS mode ( traefik). 14 | Ok so we need a certificate 15 | 16 | * GL: Create the certificate using let's encrypt 17 | 18 | * GS: Deploy traefik, show the end points configuration in values. 19 | How to configure the end points. 20 | * GS: Deploy bugs and bunny, in this case we don't use use the load-balancer 21 | option. Show the TLS configuration and the option disableNonTLSListeners. 22 | 23 | * GS: Deploy bugs and bunny ingress, explain the SNI Match. 24 | Relation between ingress and endpoints. 25 | 26 | 27 | * GS: Perf-test with for bugs, bunny and a wrong sni. 28 | 29 | 30 | 1 - traefik 31 | 2 - cert-manager 32 | 3 - make cert-manager-cloudflare-issuer 33 | 4 - make cluster-operator 34 | 5 - make rabbitmq-bunny 35 | 6 - make traefik-bunny-ingress 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | -------------------------------------------------------------------------------- /s01/e09/.env: -------------------------------------------------------------------------------- 1 | ../../.env -------------------------------------------------------------------------------- /s01/e09/.gitignore: -------------------------------------------------------------------------------- 1 | .config 2 | .envrc 3 | bin 4 | tmp 5 | -------------------------------------------------------------------------------- /s01/e09/Makefile: -------------------------------------------------------------------------------- 1 | include $(dir $(word $(words $(MAKEFILE_LIST)),$(MAKEFILE_LIST)))/../../Makefile 2 | 3 | TGIR := tgir-s01e09 4 | CLUSTER := $(TGIR)-$(USER) 5 | # Find out what versions are available: make k8s-versions 6 | # K8S versions valid at 04 December 2020 7 | CLUSTER_VERSION ?= 1.18.12-gke.1200 8 | CLUSTER_NODE_VERSION ?= 1.18.12-gke.1200 9 | CLUSTER_RELEASES ?= rapid 10 | CLUSTER_NODE_TYPE ?= n2-standard-4 11 | CLUSTER_NODES_PER_ZONE ?= 2 12 | 13 | # You may want to overwrite this with your GCP project, e.g. export GCP_PROJECT=my-project-name 14 | GCP_PROJECT ?= cf-rabbitmq-core 15 | # You may want to overwrite this with your preferred GCP region, e.g. export GCP_REGION=us-east1 16 | GCP_REGION ?= europe-west2 17 | 18 | # https://github.com/rabbitmq/cluster-operator/releases 19 | RABBITMQ_OPERATOR_VERSION := v1.2.0 20 | 21 | KUBECONFIG_DIR := $(XDG_CONFIG_HOME)/kubectl 22 | KUBECONFIG := $(KUBECONFIG_DIR)/config 23 | export KUBECONFIG 24 | 25 | RABBITMQ_DEFAULT_USER ?= $(USER) 26 | RABBITMQ_DEFAULT_PASS ?= $(TGIR) 27 | RABBITMQ_ERLANG_COOKIE ?= $(CLUSTER) 28 | 29 | CLOUDSDK_CONFIG := $(XDG_CONFIG_HOME)/gcloud/configurations/config_default 30 | export CLOUDSDK_CONFIG 31 | $(CLOUDSDK_CONFIG): $(GCLOUD) 32 | $(GCLOUD) auth login \ 33 | && $(GCLOUD) config set project $(GCP_PROJECT) \ 34 | && $(GCLOUD) config set compute/region $(GCP_REGION) 35 | 36 | $(KUBECONFIG_DIR): 37 | mkdir -p $(@) 38 | $(KUBECONFIG): | $(KUBECTL) $(KUBECONFIG_DIR) $(CLOUDSDK_CONFIG) 39 | $(GCLOUD) container clusters get-credentials $(CLUSTER) 40 | 41 | .PHONY: k9s 42 | k9s: | $(K9S) $(KUBECONFIG) ## Interact with our K8S cluster via a terminal UI 43 | $(K9S) --all-namespaces 44 | 45 | .PHONY: k9 46 | k9: | $(K9S) $(KUBECONFIG) 47 | $(K9S) --namespace default --headless 48 | 49 | define ENV 50 | export PATH=$(LOCAL_BIN):$$PATH 51 | export GCP_PROJECT="$(GCP_PROJECT)" 52 | export GCP_REGION="$(GCP_REGION)" 53 | export KUBECONFIG="$(KUBECONFIG)" 54 | export XDG_CONFIG_HOME="$(XDG_CONFIG_HOME)" 55 | export CLOUDSDK_CONFIG="$(CLOUDSDK_CONFIG)" 56 | unalias k 2>/dev/null; alias k=kubectl 57 | unalias m 2>/dev/null; alias m=make 58 | endef 59 | export ENV 60 | .PHONY: env 61 | env:: | $(CLOUDSDK_CONFIG) $(KUBECONFIG_DIR) $(KUBECTL) ## Configure shell env - eval "$(make env)" OR source .env 62 | @echo "$$ENV" 63 | 64 | define LIST_INSTANCES 65 | $(GCLOUD) compute instances list --filter='name ~ $(CLUSTER)' 66 | endef 67 | instances: | $(CLOUDSDK_CONFIG) ## List all instances 68 | $(LIST_INSTANCES) 69 | 70 | watch-instances: | $(CLOUDSDK_CONFIG) ## Watch all instances 71 | watch -c "$(LIST_INSTANCES)" 72 | 73 | watch-nodes: | $(KUBECONFIG) ## Watch all K8S nodes 74 | watch -c "$(KUBECTL) get nodes --output=wide" 75 | 76 | disks: | $(CLOUDSDK_CONFIG) ## List all disks 77 | $(GCLOUD) compute disks list --filter='name ~ $(CLUSTER)' 78 | 79 | .PHONY: k8s-versions 80 | k8s-versions: | $(CLOUDSDK_CONFIG) ## List all available K8S versions on GCP (GKE) 81 | $(GCLOUD) container get-server-config 82 | 83 | .PHONY: k8s 84 | k8s: | $(CLOUDSDK_CONFIG) ## Create a managed K8S cluster on GCP (GKE) - up to 4 minutes 85 | $(GCLOUD) container clusters describe $(CLUSTER) \ 86 | || time $(GCLOUD) container clusters create $(CLUSTER) \ 87 | --release-channel $(CLUSTER_RELEASES) \ 88 | --cluster-version $(CLUSTER_VERSION) \ 89 | --node-version $(CLUSTER_NODE_VERSION) \ 90 | --machine-type $(CLUSTER_NODE_TYPE) \ 91 | --num-nodes $(CLUSTER_NODES_PER_ZONE) \ 92 | --enable-shielded-nodes \ 93 | --disk-type "pd-ssd" \ 94 | --disk-size "100" \ 95 | --enable-ip-alias \ 96 | --enable-autoupgrade \ 97 | --enable-autorepair \ 98 | --max-surge-upgrade $(CLUSTER_NODES_PER_ZONE) \ 99 | --max-unavailable-upgrade 0 \ 100 | --metadata disable-legacy-endpoints=true \ 101 | --no-enable-master-authorized-networks \ 102 | --addons "HorizontalPodAutoscaling,HttpLoadBalancing" 103 | 104 | .PHONY: all 105 | all: | k8s monitoring-stack chaos-operator rabbitmq-operator ## Create the cluster & stacks needed for the episode in GKE - Chaos Mesh, Grafana, Prometheus & RabbitMQ Operator 106 | 107 | .PHONY: base 108 | base: | k8s monitoring-stack rabbitmq-operator ## Create the cluster & all stacks needed for the episode in GKE, except for Chaos Mesh 109 | 110 | .PHONY: k8s-help 111 | k8s-help: | $(CLOUDSDK_CONFIG) ## List all options available when creating a managed K8S cluster on GCP (GKE) 112 | $(GCLOUD) container clusters create --help 113 | 114 | .PHONY: k8s-ls 115 | k8s-ls: | $(CLOUDSDK_CONFIG) ## List all GKE clusters running on GCP 116 | $(GCLOUD) container clusters list 117 | 118 | .PHONY: k8s-rm 119 | k8s-rm: | $(CLOUDSDK_CONFIG) ## Delete our GKE cluster 120 | $(GCLOUD) container clusters delete $(CLUSTER) 121 | 122 | # https://github.com/chaos-mesh/chaos-mesh/releases 123 | CHAOS_MESH_VERSION := v1.0.2 124 | .PHONY: chaos-operator 125 | chaos-operator: | $(KUBECONFIG) ## Install Chaos Mesh Operator 126 | $(CURL) -sSL https://mirrors.chaos-mesh.org/$(CHAOS_MESH_VERSION)/install.sh | bash 127 | 128 | .PHONY: teardown-chaos-operator 129 | teardown-chaos-operator: | $(KUBECONFIG) ## Remove Chaos Mesh Operator 130 | $(CURL) -sSL https://mirrors.chaos-mesh.org/$(CHAOS_MESH_VERSION)/install.sh | bash -s -- --template | $(KUBECTL) delete -f - 131 | 132 | # helm search repo --versions prometheus-community/kube-prometheus-stack 133 | KUBE_PROMETHEUS_STACK_VERSION ?= 12.2.3 134 | .PHONY: monitoring-stack 135 | monitoring-stack: | $(KUBECONFIG) $(HELM) ## Integrate Prometheus & Grafana with K8S, including system metrics 136 | $(KUBECTL) create namespace monitoring 137 | $(HELM) repo add prometheus-community https://prometheus-community.github.io/helm-charts 138 | $(HELM) upgrade prometheus prometheus-community/kube-prometheus-stack \ 139 | --install \ 140 | --namespace monitoring \ 141 | --version $(KUBE_PROMETHEUS_STACK_VERSION) \ 142 | --values k8s/monitoring-stack/values.yml 143 | $(KUBECTL) apply \ 144 | --filename k8s/monitoring-stack/traefik-servicemonitor.yml \ 145 | --filename k8s/monitoring-stack/dashboards 146 | 147 | .PHONY: teardown-monitoring-stack 148 | teardown-monitoring-stack: | $(KUBECONFIG) $(HELM) ## Teardown the whole monitoring stack 149 | $(KUBECTL) delete --filename k8s/monitoring-stack/dashboards --ignore-not-found 150 | $(KUBECTL) delete --filename k8s/monitoring-stack/traefik-servicemonitor.yml --ignore-not-found 151 | $(HELM) uninstall --namespace monitoring prometheus 152 | $(HELM) repo remove prometheus-community 153 | $(KUBECTL) delete namespace monitoring --ignore-not-found 154 | 155 | .PHONY: rabbitmq-operator 156 | rabbitmq-operator: | $(KUBECONFIG) ## Install RabbitMQ Cluster Operator into K8S 157 | $(KUBECTL) apply --filename https://github.com/rabbitmq/cluster-operator/releases/download/$(RABBITMQ_OPERATOR_VERSION)/cluster-operator.yml 158 | 159 | .PHONY: teardown-rabbitmq-operator 160 | teardown-rabbitmq-operator: | $(KUBECONFIG) ## Teardown the RabbitMQ Cluster Operator 161 | $(KUBECTL) delete --ignore-not-found --filename https://github.com/rabbitmq/cluster-operator/releases/download/$(RABBITMQ_OPERATOR_VERSION)/cluster-operator.yml 162 | 163 | .PHONY: rabbitmq-production-cluster 164 | rabbitmq-production-cluster: | rabbitmq-operator ## Install the production-ready RabbitMQ cluster 165 | $(KUBECTL) apply --filename k8s/rabbitmq/ssd-gke.yaml 166 | $(KUBECTL) apply --filename k8s/rabbitmq/pod-disruption-budget.yaml 167 | $(KUBECTL) apply --filename k8s/rabbitmq/clusters/partition-ignore.yaml 168 | 169 | .PHONY: teardown-production-cluster 170 | teardown-production-cluster: | $(KUBECTL) ## Teardown the production-ready RabbitMQ cluster 171 | $(KUBECTL) delete --ignore-not-found --filename k8s/rabbitmq/clusters/partition-ignore.yaml 172 | $(KUBECTL) delete --ignore-not-found --filename k8s/rabbitmq/ssd-gke.yaml 173 | $(KUBECTL) delete --ignore-not-found --filename k8s/rabbitmq/pod-disruption-budget.yaml 174 | 175 | .PHONY: rabbitmq-pause-minority-cluster 176 | rabbitmq-pause-minority-cluster: | rabbitmq-operator ## Install the production-ready RabbitMQ cluster with pause_minority partition handling 177 | $(KUBECTL) apply --filename k8s/rabbitmq/ssd-gke.yaml 178 | $(KUBECTL) apply --filename k8s/rabbitmq/pod-disruption-budget.yaml 179 | $(KUBECTL) apply --filename k8s/rabbitmq/clusters/partition-pause-minority.yaml 180 | 181 | .PHONY: teardown-pause-minority-cluster 182 | teardown-pause-minority-cluster: | $(KUBECTL) ## Teardown the pause_minority production-ready RabbitMQ cluster 183 | $(KUBECTL) delete --ignore-not-found --filename k8s/rabbitmq/clusters/partition-pause-minority.yaml 184 | $(KUBECTL) delete --ignore-not-found --filename k8s/rabbitmq/ssd-gke.yaml 185 | $(KUBECTL) delete --ignore-not-found --filename k8s/rabbitmq/pod-disruption-budget.yaml 186 | 187 | .PHONY: quorum-clients 188 | quorum-clients: | $(KUBECONFIG) ## Create RabbitMQ clients to start 1000 quorum queues 189 | $(KUBECTL) apply --filename k8s/rabbitmq/clients/perf-test-quorum.yaml 190 | 191 | .PHONY: teardown-quorum-clients 192 | teardown-quorum-clients: | $(KUBECONFIG) ## Delete quorum queue RabbitMQ clients 193 | $(KUBECTL) delete --ignore-not-found --filename k8s/rabbitmq/clients/perf-test-quorum.yaml 194 | 195 | .PHONY: rebalance-queues 196 | rebalance-queues: | $(KUBECONFIG) ## Exec into RabbitMQ Pod to rebalance queue leaders across cluster 197 | $(KUBECTL) exec svc/production-ready -- rabbitmq-queues rebalance all 198 | 199 | .PHONY: rabbitmq-observer 200 | rabbitmq-observer: | $(KUBECONFIG) ## Exec into RabbitMQ Pod to launch the RabbitMQ diagnostics observer 201 | $(KUBECTL) exec -it svc/production-ready -- rabbitmq-diagnostics observer 202 | 203 | .PHONY: clear-chaos 204 | clear-chaos: | $(KUBECONFIG) ## Clear any Chaos Mesh events from the cluster 205 | $(KUBECTL) delete --ignore-not-found --filename k8s/chaos-experiments 206 | 207 | .PHONY: chaos-az-latency 208 | chaos-az-latency: | $(KUBECONFIG) ## Introduce 1s of latency to&from a random pod in the cluster 209 | $(KUBECTL) apply --filename k8s/chaos-experiments/cluster-latency.yaml 210 | 211 | .PHONY: chaos-az-partition 212 | chaos-az-partition: | $(KUBECONFIG) ## Create a network partition seperating one RabbitMQ node completely from the other two 213 | $(KUBECTL) apply --filename k8s/chaos-experiments/cluster-partition.yaml 214 | 215 | .PHONY: chaos-intra-node-partition 216 | chaos-intra-node-partition: | $(KUBECONFIG) ## Create a network partition only between two of the nodes, leaving other connections intact 217 | $(KUBECTL) apply --filename k8s/chaos-experiments/intra-node-partition.yaml 218 | 219 | .PHONY: chaos-cpu-stealing 220 | chaos-cpu-stealing: | $(KUBECONFIG) ## Cause CPU pressure in a random RabbitMQ pod, simulating CPU stealing 221 | $(KUBECTL) apply --filename k8s/chaos-experiments/cpu-stealing.yaml 222 | 223 | .PHONY: chaos-memory-filling 224 | chaos-memory-filling: | $(KUBECONFIG) ## Cause memory pressure in a random RabbitMQ pod, reducing available memory for RabbitMQ 225 | $(KUBECTL) apply --filename k8s/chaos-experiments/memory-filling.yaml 226 | 227 | .PHONY: chaos-slow-disk 228 | chaos-slow-disk: | $(KUBECONFIG) ## Add latency to file I/O operations to simulate a slow disk on a random RabbitMQ pod 229 | $(KUBECTL) apply --filename k8s/chaos-experiments/disk-latency.yaml 230 | 231 | -------------------------------------------------------------------------------- /s01/e09/README.md: -------------------------------------------------------------------------------- 1 | # TGIR S01E09: Testing RabbitMQ Resiliency with Chaos Mesh 2 | 3 | * Proposed via [rabbitmq/tgir#19](https://github.com/rabbitmq/tgir/issues/19) 4 | * Hosted by [@corodotdev](https://twitter.com/corodotdev) & [@gerhardlazu](https://twitter.com/gerhardlazu) 5 | * Published on: 2020-12-31 6 | * Video: https://www.youtube.com/watch?v=y2HAJBiXsw0 7 | 8 | 9 | 10 | How does RabbitMQ handle network latency? 11 | 12 | What about a clean network partition? 13 | 14 | And a partial network partition, or [Byzantine failure](https://blog.cloudflare.com/a-byzantine-failure-in-the-real-world/)? 15 | 16 | We have at our disposal a wide variety of tooling for the Kubernetes infrastructure that will let us make new discoveries about the behaviour of RabbitMQ. 17 | 18 | In this episode, we give you all the tools to set you up to make these discoveries yourself, and show you some of the interesting behaviours that we have discovered. 19 | 20 | 21 | 22 | ## LINKS 23 | 24 | * [Chaos Mesh](https://chaos-mesh.org/) - Chaos testing framework for Kubernetes clusters by the CNCF 25 | * [RabbitMQ Cluster Kubernetes Operator](https://github.com/rabbitmq/cluster-operator) - to easily spin up RabbitMQ Clusters in Kubernetes 26 | * [TGIR S01E07](https://www.youtube.com/watch?v=NWISW6AwpOE) - Setting up the monitoring framework used in this episode 27 | * [Cloudflare blog about their partial network partitions](https://blog.cloudflare.com/a-byzantine-failure-in-the-real-world/) 28 | * [FOLLOW-UP: Chaos Mesh Community Meetup - Testing RabbitMQ Resiliency with Chaos Mesh - Jan 28, 2021](https://twitter.com/chaos_mesh/status/1354114146443423746) 29 | 30 | 31 | 32 | ## MAKE TARGETS 33 | 34 | ``` 35 | all Create the cluster & stacks needed for the episode in GKE - Chaos Mesh, Grafana, Prometheus & RabbitMQ Operator 36 | base Create the cluster & all stacks needed for the episode in GKE, except for Chaos Mesh 37 | chaos-az-latency Introduce 1s of latency to&from a random pod in the cluster 38 | chaos-az-partition Create a network partition seperating one RabbitMQ node completely from the other two 39 | chaos-cpu-stealing Cause CPU pressure in a random RabbitMQ pod, simulating CPU stealing 40 | chaos-intra-node-partition Create a network partition only between two of the nodes, leaving other connections intact 41 | chaos-memory-filling Cause memory pressure in a random RabbitMQ pod, reducing available memory for RabbitMQ 42 | chaos-operator Install Chaos Mesh Operator 43 | chaos-slow-disk Add latency to file I/O operations to simulate a slow disk on a random RabbitMQ pod 44 | clear-chaos Clear any Chaos Mesh events from the cluster 45 | disks List all disks 46 | env Configure shell env - eval "$(make env)" OR source .env 47 | instances List all instances 48 | k8s Create a managed K8S cluster on GCP (GKE) - up to 4 minutes 49 | k8s-help List all options available when creating a managed K8S cluster on GCP (GKE) 50 | k8s-ls List all GKE clusters running on GCP 51 | k8s-rm Delete our GKE cluster 52 | k8s-versions List all available K8S versions on GCP (GKE) 53 | k9s Interact with our K8S cluster via a terminal UI 54 | monitoring-stack Integrate Prometheus & Grafana with K8S, including system metrics 55 | quorum-clients Create RabbitMQ clients to start 1000 quorum queues 56 | rabbitmq-observer Exec into RabbitMQ Pod to launch the RabbitMQ diagnostics observer 57 | rabbitmq-operator Install RabbitMQ Cluster Operator into K8S 58 | rabbitmq-pause-minority-cluster Install the production-ready RabbitMQ cluster with pause_minority partition handling 59 | rabbitmq-production-cluster Install the production-ready RabbitMQ cluster 60 | rebalance-queues Exec into RabbitMQ Pod to rebalance queue leaders across cluster 61 | teardown-chaos-operator Remove Chaos Mesh Operator 62 | teardown-monitoring-stack Teardown the whole monitoring stack 63 | teardown-pause-minority-cluster Teardown the pause_minority production-ready RabbitMQ cluster 64 | teardown-production-cluster Teardown the production-ready RabbitMQ cluster 65 | teardown-quorum-clients Delete quorum queue RabbitMQ clients 66 | teardown-rabbitmq-operator Teardown the RabbitMQ Cluster Operator 67 | watch-instances Watch all instances 68 | watch-nodes Watch all K8S nodes 69 | ``` 70 | 71 | 72 | 73 | ## FOLLOW-UP 74 | 75 | And here is a follow-up with the Chaos Mesh Community from January 28, 2021: https://twitter.com/chaos_mesh/status/1354114146443423746 76 | 77 | 78 | -------------------------------------------------------------------------------- /s01/e09/k8s/chaos-experiments/cluster-latency.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: chaos-mesh.org/v1alpha1 2 | kind: NetworkChaos 3 | metadata: 4 | name: cluster-latency 5 | namespace: chaos-testing 6 | spec: 7 | action: delay 8 | mode: one 9 | selector: 10 | namespaces: 11 | - default 12 | labelSelectors: 13 | "app.kubernetes.io/component": "rabbitmq" 14 | direction: both 15 | target: 16 | selector: 17 | namespaces: 18 | - default 19 | labelSelectors: 20 | "app.kubernetes.io/component": "rabbitmq" 21 | mode: all 22 | delay: 23 | latency: "1000ms" 24 | -------------------------------------------------------------------------------- /s01/e09/k8s/chaos-experiments/cluster-partition.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: chaos-mesh.org/v1alpha1 2 | kind: NetworkChaos 3 | metadata: 4 | name: partition-single-node 5 | namespace: chaos-testing 6 | spec: 7 | action: partition 8 | mode: one 9 | selector: 10 | pods: 11 | default: 12 | - production-ready-server-1 13 | direction: both 14 | target: 15 | selector: 16 | pods: 17 | default: 18 | - production-ready-server-0 19 | - production-ready-server-2 20 | mode: all 21 | -------------------------------------------------------------------------------- /s01/e09/k8s/chaos-experiments/cpu-stealing.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: chaos-mesh.org/v1alpha1 2 | kind: StressChaos 3 | metadata: 4 | name: cpu-stealing 5 | namespace: chaos-testing 6 | spec: 7 | mode: one 8 | selector: 9 | namespaces: 10 | - default 11 | labelSelectors: 12 | "app.kubernetes.io/component": "rabbitmq" 13 | stressors: 14 | cpu: 15 | workers: 2 16 | load: 80 17 | -------------------------------------------------------------------------------- /s01/e09/k8s/chaos-experiments/disk-latency.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: chaos-mesh.org/v1alpha1 2 | kind: IoChaos 3 | metadata: 4 | name: disk-latency 5 | namespace: chaos-testing 6 | spec: 7 | action: latency 8 | mode: one 9 | selector: 10 | namespaces: 11 | - default 12 | labelSelectors: 13 | "app.kubernetes.io/component": "rabbitmq" 14 | delay: "1000ms" 15 | volumePath: "/var/lib/rabbitmq/mnesia/" 16 | path: "/var/lib/rabbitmq/mnesia/**/*" 17 | -------------------------------------------------------------------------------- /s01/e09/k8s/chaos-experiments/intra-node-partition.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: chaos-mesh.org/v1alpha1 2 | kind: NetworkChaos 3 | metadata: 4 | name: partition-single-node 5 | namespace: chaos-testing 6 | spec: 7 | action: partition 8 | mode: one 9 | selector: 10 | pods: 11 | default: 12 | - production-ready-server-1 13 | direction: both 14 | target: 15 | selector: 16 | pods: 17 | default: 18 | - production-ready-server-2 19 | mode: all 20 | -------------------------------------------------------------------------------- /s01/e09/k8s/chaos-experiments/kill-rmq-pod-every-5m.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: chaos-mesh.org/v1alpha1 2 | kind: PodChaos 3 | metadata: 4 | name: kill-rmq-pod-every-5m 5 | namespace: chaos-testing 6 | spec: 7 | action: pod-kill 8 | mode: one 9 | selector: 10 | labelSelectors: 11 | "app.kubernetes.io/component": "rabbitmq" 12 | scheduler: 13 | cron: "@every 5m" 14 | -------------------------------------------------------------------------------- /s01/e09/k8s/chaos-experiments/memory-filling.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: chaos-mesh.org/v1alpha1 2 | kind: StressChaos 3 | metadata: 4 | name: memory-filling 5 | namespace: chaos-testing 6 | spec: 7 | mode: one 8 | selector: 9 | namespaces: 10 | - default 11 | labelSelectors: 12 | "app.kubernetes.io/component": "rabbitmq" 13 | stressors: 14 | memory: 15 | workers: 3 16 | load: "1Gi" 17 | -------------------------------------------------------------------------------- /s01/e09/k8s/monitoring-stack/traefik-servicemonitor.yml: -------------------------------------------------------------------------------- 1 | apiVersion: monitoring.coreos.com/v1 2 | kind: ServiceMonitor 3 | metadata: 4 | labels: 5 | app: traefik 6 | release: prometheus 7 | name: traefik 8 | namespace: monitoring 9 | spec: 10 | endpoints: 11 | - port: metrics 12 | namespaceSelector: 13 | matchNames: 14 | - kube-system 15 | selector: 16 | matchLabels: 17 | app: traefik 18 | 19 | -------------------------------------------------------------------------------- /s01/e09/k8s/rabbitmq/clients/perf-test-classic-queues-second-client.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: production-perf-test-second 5 | spec: 6 | replicas: 1 7 | selector: 8 | matchLabels: 9 | app: rabbitmq-perf-test-second 10 | template: 11 | metadata: 12 | labels: 13 | app: rabbitmq-perf-test-second 14 | annotations: 15 | prometheus.io/port: "8080" 16 | prometheus.io/scrape: "true" 17 | spec: 18 | containers: 19 | - name: perftest 20 | # https://hub.docker.com/r/pivotalrabbitmq/perf-test/tags 21 | image: pivotalrabbitmq/perf-test:2.12.0 22 | resources: 23 | limits: 24 | cpu: 2 25 | memory: 3G 26 | requests: 27 | cpu: 1 28 | memory: 1G 29 | command: 30 | - /bin/bash 31 | - -c 32 | args: 33 | - |- 34 | bin/runjava com.rabbitmq.perf.PerfTest \ 35 | --uri "amqp://$RABBITMQ_USER:$RABBITMQ_PASS@$RABBITMQ_SERVICE:5672/%2f" 36 | env: 37 | - name: RABBITMQ_USER 38 | valueFrom: 39 | secretKeyRef: 40 | name: production-ready-default-user 41 | key: username 42 | - name: RABBITMQ_PASS 43 | valueFrom: 44 | secretKeyRef: 45 | name: production-ready-default-user 46 | key: password 47 | - name: RABBITMQ_SERVICE 48 | value: production-ready 49 | 50 | - name: PRODUCER_SCHEDULER_THREADS 51 | value: "10" 52 | - name: PRODUCERS 53 | value: "100" 54 | - name: PRODUCER_RANDOM_START_DELAY 55 | value: "60" 56 | - name: RATE 57 | value: "10" 58 | - name: ROUTING_KEY 59 | value: "production-non-durable" 60 | - name: SIZE 61 | value: "100" 62 | 63 | - name: CONSUMERS_THREAD_POOLS 64 | value: "10" 65 | - name: CONSUMERS 66 | value: "100" 67 | 68 | - name: HEARTBEAT_SENDER_THREADS 69 | value: "500" 70 | - name: NIO_THREAD_POOL 71 | value: "500" 72 | 73 | - name: QUEUE_PATTERN 74 | value: "production-non-durable-%d" 75 | - name: QUEUE_PATTERN_FROM 76 | value: "100" 77 | - name: QUEUE_PATTERN_TO 78 | value: "200" 79 | - name: QUEUE_ARGS 80 | value: "x-max-length=100" 81 | - name: AUTO_DELETE 82 | value: "false" 83 | 84 | - name: METRICS_PROMETHEUS 85 | value: "true" 86 | -------------------------------------------------------------------------------- /s01/e09/k8s/rabbitmq/clients/perf-test-classic-queues.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: production-perf-test 5 | spec: 6 | replicas: 1 7 | selector: 8 | matchLabels: 9 | app: rabbitmq-perf-test 10 | template: 11 | metadata: 12 | labels: 13 | app: rabbitmq-perf-test 14 | annotations: 15 | prometheus.io/port: "8080" 16 | prometheus.io/scrape: "true" 17 | spec: 18 | containers: 19 | - name: perftest 20 | # https://hub.docker.com/r/pivotalrabbitmq/perf-test/tags 21 | image: pivotalrabbitmq/perf-test:2.12.0 22 | resources: 23 | limits: 24 | cpu: 2 25 | memory: 3G 26 | requests: 27 | cpu: 1 28 | memory: 1G 29 | command: 30 | - /bin/bash 31 | - -c 32 | args: 33 | - |- 34 | bin/runjava com.rabbitmq.perf.PerfTest \ 35 | --uri "amqp://$RABBITMQ_USER:$RABBITMQ_PASS@$RABBITMQ_SERVICE:5672/%2f" 36 | env: 37 | - name: RABBITMQ_USER 38 | valueFrom: 39 | secretKeyRef: 40 | name: production-ready-default-user 41 | key: username 42 | - name: RABBITMQ_PASS 43 | valueFrom: 44 | secretKeyRef: 45 | name: production-ready-default-user 46 | key: password 47 | - name: RABBITMQ_SERVICE 48 | value: production-ready 49 | 50 | - name: PRODUCER_SCHEDULER_THREADS 51 | value: "10" 52 | - name: PRODUCERS 53 | value: "100" 54 | - name: PRODUCER_RANDOM_START_DELAY 55 | value: "60" 56 | - name: RATE 57 | value: "10" 58 | - name: ROUTING_KEY 59 | value: "production-non-durable" 60 | - name: SIZE 61 | value: "100" 62 | 63 | - name: CONSUMERS_THREAD_POOLS 64 | value: "10" 65 | - name: CONSUMERS 66 | value: "100" 67 | 68 | - name: HEARTBEAT_SENDER_THREADS 69 | value: "500" 70 | - name: NIO_THREAD_POOL 71 | value: "500" 72 | 73 | - name: QUEUE_PATTERN 74 | value: "production-non-durable-%d" 75 | - name: QUEUE_PATTERN_FROM 76 | value: "1" 77 | - name: QUEUE_PATTERN_TO 78 | value: "100" 79 | - name: QUEUE_ARGS 80 | value: "x-max-length=100" 81 | - name: AUTO_DELETE 82 | value: "false" 83 | 84 | - name: METRICS_PROMETHEUS 85 | value: "true" 86 | -------------------------------------------------------------------------------- /s01/e09/k8s/rabbitmq/clients/perf-test-quorum.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: quorum-rabbitmq-perf-test 5 | spec: 6 | replicas: 1 7 | selector: 8 | matchLabels: 9 | app: quorum-rabbitmq-perf-test 10 | template: 11 | metadata: 12 | labels: 13 | app: quorum-rabbitmq-perf-test 14 | annotations: 15 | prometheus.io/port: "8080" 16 | prometheus.io/scrape: "true" 17 | spec: 18 | containers: 19 | - name: perftest 20 | # https://hub.docker.com/r/pivotalrabbitmq/perf-test/tags 21 | image: pivotalrabbitmq/perf-test:2.12.0 22 | resources: 23 | limits: 24 | cpu: 2 25 | memory: 2G 26 | requests: 27 | cpu: 1 28 | memory: 1G 29 | command: 30 | - /bin/bash 31 | - -c 32 | args: 33 | - |- 34 | bin/runjava com.rabbitmq.perf.PerfTest \ 35 | --uri "amqp://$RABBITMQ_USER:$RABBITMQ_PASS@$RABBITMQ_SERVICE:5672/%2f" 36 | env: 37 | - name: RABBITMQ_USER 38 | valueFrom: 39 | secretKeyRef: 40 | name: production-ready-default-user 41 | key: username 42 | - name: RABBITMQ_PASS 43 | valueFrom: 44 | secretKeyRef: 45 | name: production-ready-default-user 46 | key: password 47 | - name: RABBITMQ_SERVICE 48 | value: production-ready 49 | 50 | - name: PRODUCER_SCHEDULER_THREADS 51 | value: "10" 52 | - name: PRODUCERS 53 | value: "100" 54 | - name: PRODUCER_RANDOM_START_DELAY 55 | value: "30" 56 | - name: RATE 57 | value: "10" 58 | - name: ROUTING_KEY 59 | value: "production-non-durable" 60 | - name: SIZE 61 | value: "10000" 62 | 63 | - name: CONSUMERS_THREAD_POOLS 64 | value: "10" 65 | - name: CONSUMERS 66 | value: "100" 67 | 68 | - name: HEARTBEAT_SENDER_THREADS 69 | value: "500" 70 | - name: NIO_THREAD_POOL 71 | value: "500" 72 | 73 | - name: METRICS_PROMETHEUS 74 | value: "true" 75 | - name: CONFIRM 76 | value: "1" 77 | - name: JSON_BODY 78 | value: "true" 79 | - name: SIZE 80 | value: "100" 81 | - name: FLAG 82 | value: "persistent" 83 | - name: ROUTING_KEY 84 | value: "quorum" 85 | 86 | - name: QUEUE_PATTERN 87 | value: "quorum-%d" 88 | - name: QUEUE_PATTERN_FROM 89 | value: "1" 90 | - name: QUEUE_PATTERN_TO 91 | value: "100" 92 | - name: QUEUE_ARGS 93 | value: "x-max-length=1000,x-queue-type=quorum,x-max-in-memory-bytes=10000000" 94 | - name: AUTO_DELETE 95 | value: "false" 96 | -------------------------------------------------------------------------------- /s01/e09/k8s/rabbitmq/clusters/partition-autoheal.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rabbitmq.com/v1beta1 2 | kind: RabbitmqCluster 3 | metadata: 4 | name: production-ready 5 | spec: 6 | replicas: 3 7 | resources: 8 | requests: 9 | cpu: 2 10 | memory: 4Gi 11 | limits: 12 | cpu: 2 13 | memory: 4Gi 14 | rabbitmq: 15 | additionalConfig: | 16 | cluster_partition_handling = autoheal 17 | vm_memory_high_watermark_paging_ratio = 0.99 18 | disk_free_limit.relative = 1.0 19 | persistence: 20 | storageClassName: ssd 21 | storage: "500Gi" 22 | override: 23 | statefulSet: 24 | spec: 25 | template: 26 | spec: 27 | containers: [] 28 | topologySpreadConstraints: 29 | - maxSkew: 1 30 | topologyKey: "topology.kubernetes.io/zone" 31 | whenUnsatisfiable: DoNotSchedule 32 | labelSelector: 33 | matchLabels: 34 | app.kubernetes.io/component: rabbitmq 35 | -------------------------------------------------------------------------------- /s01/e09/k8s/rabbitmq/clusters/partition-ignore.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rabbitmq.com/v1beta1 2 | kind: RabbitmqCluster 3 | metadata: 4 | name: production-ready 5 | spec: 6 | replicas: 3 7 | resources: 8 | requests: 9 | cpu: 2 10 | memory: 4Gi 11 | limits: 12 | cpu: 2 13 | memory: 4Gi 14 | rabbitmq: 15 | additionalConfig: | 16 | cluster_partition_handling = ignore 17 | vm_memory_high_watermark_paging_ratio = 0.99 18 | disk_free_limit.relative = 1.0 19 | persistence: 20 | storageClassName: ssd 21 | storage: "500Gi" 22 | override: 23 | statefulSet: 24 | spec: 25 | template: 26 | spec: 27 | containers: [] 28 | topologySpreadConstraints: 29 | - maxSkew: 1 30 | topologyKey: "topology.kubernetes.io/zone" 31 | whenUnsatisfiable: DoNotSchedule 32 | labelSelector: 33 | matchLabels: 34 | app.kubernetes.io/component: rabbitmq 35 | -------------------------------------------------------------------------------- /s01/e09/k8s/rabbitmq/clusters/partition-pause-minority.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rabbitmq.com/v1beta1 2 | kind: RabbitmqCluster 3 | metadata: 4 | name: production-ready 5 | spec: 6 | replicas: 3 7 | resources: 8 | requests: 9 | cpu: 2 10 | memory: 4Gi 11 | limits: 12 | cpu: 2 13 | memory: 4Gi 14 | rabbitmq: 15 | additionalConfig: | 16 | cluster_partition_handling = pause_minority 17 | vm_memory_high_watermark_paging_ratio = 0.99 18 | disk_free_limit.relative = 1.0 19 | persistence: 20 | storageClassName: ssd 21 | storage: "500Gi" 22 | override: 23 | statefulSet: 24 | spec: 25 | template: 26 | spec: 27 | containers: [] 28 | topologySpreadConstraints: 29 | - maxSkew: 1 30 | topologyKey: "topology.kubernetes.io/zone" 31 | whenUnsatisfiable: DoNotSchedule 32 | labelSelector: 33 | matchLabels: 34 | app.kubernetes.io/component: rabbitmq 35 | -------------------------------------------------------------------------------- /s01/e09/k8s/rabbitmq/pod-disruption-budget.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: policy/v1beta1 2 | kind: PodDisruptionBudget 3 | metadata: 4 | name: production-ready-rabbitmq 5 | spec: 6 | maxUnavailable: 1 7 | selector: 8 | matchLabels: 9 | app.kubernetes.io/name: production-ready 10 | -------------------------------------------------------------------------------- /s01/e09/k8s/rabbitmq/ssd-gke.yaml: -------------------------------------------------------------------------------- 1 | # This example is GKE specific, other Kubernetes deployments will have a different `storageClass` for SSD disks. 2 | # 3 | # https://cloud.google.com/kubernetes-engine/docs/how-to/persistent-volumes/ssd-pd 4 | # https://cloud.google.com/compute/docs/disks/performance#ssd_persistent_disk 5 | apiVersion: storage.k8s.io/v1 6 | kind: StorageClass 7 | metadata: 8 | name: ssd 9 | provisioner: kubernetes.io/gce-pd 10 | parameters: 11 | type: pd-ssd 12 | 13 | -------------------------------------------------------------------------------- /s01/e09/rabbitmq-chaos-mesh-community-meetup-january-2021.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rabbitmq/tgir/87c753ae5bffa8cb8b321ca58297858153adeb6f/s01/e09/rabbitmq-chaos-mesh-community-meetup-january-2021.jpg -------------------------------------------------------------------------------- /s01/e09/video.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rabbitmq/tgir/87c753ae5bffa8cb8b321ca58297858153adeb6f/s01/e09/video.jpg --------------------------------------------------------------------------------