├── content ├── 00-title.md ├── 92-references.md ├── 03-toc.md ├── 91-lists.md ├── 90-appendix.md ├── 01-affirmation.md ├── 02-abstract.md ├── 10-motivation.md ├── 70-conclusion.md ├── bibliography.bib ├── 30-requirements.md ├── 40-design.md ├── 60-evaluation.md └── 50-implementation.md ├── hack ├── down.sh ├── make.sh ├── up.sh └── shell.sh ├── .gitignore ├── pandoc ├── filters │ ├── remove-date.sh │ ├── count-words.lua │ ├── pandoc-minted.py │ └── pretty-tables.py ├── fonts │ ├── JetBrainsMono-Bold.ttf │ ├── JetBrainsMono-Thin.ttf │ ├── JetBrainsMono-ExtraBold.ttf │ ├── JetBrainsMono-Italic.ttf │ ├── JetBrainsMono-Light.ttf │ ├── JetBrainsMono-Medium.ttf │ ├── JetBrainsMono-Regular.ttf │ ├── JetBrainsMono-SemiBold.ttf │ ├── JetBrainsMono-BoldItalic.ttf │ ├── JetBrainsMono-ExtraLight.ttf │ ├── JetBrainsMono-LightItalic.ttf │ ├── JetBrainsMono-MediumItalic.ttf │ ├── JetBrainsMono-SemiBoldItalic.ttf │ ├── JetBrainsMono-ThinItalic.ttf │ ├── JetBrainsMono-ExtraBoldItalic.ttf │ └── JetBrainsMono-ExtraLightItalic.ttf ├── .latexmkrc ├── metadata.yaml ├── defaults-latex.yaml ├── defaults.yaml ├── highlight-style.theme ├── includes │ └── header.tex └── ieee.csl ├── assets ├── dhbw-cas.pdf ├── design-overview.pdf ├── sample-website.png ├── cluster-architecture.pdf ├── cluster-architecture.png ├── dashboard-sharding.png ├── controller-components.jpeg ├── dashboard-webhosting.png └── dashboard-controller-runtime.png ├── .gitattributes ├── docker-compose.yaml ├── results ├── common.py ├── base-websites-plot.py ├── base-cpu-plot.py ├── base-memory-plot.py ├── base.yaml ├── base-network-plot.py └── base │ ├── singleton-memory.csv │ ├── singleton-network_transmit.csv │ ├── singleton-network_receive.csv │ ├── singleton-cpu.csv │ ├── sharded-websites_per_shard.csv │ ├── sharded-memory.csv │ ├── sharded-network_receive.csv │ ├── sharded-network_transmit.csv │ └── sharded-cpu.csv ├── .github └── workflows │ └── build.yaml ├── .run ├── Plots.run.xml ├── Clean Plots.run.xml ├── Build.run.xml └── Clean.run.xml ├── draw ├── cluster-architecture.drawio └── design-overview.drawio ├── Makefile └── README.md /content/00-title.md: -------------------------------------------------------------------------------- 1 | \newpage 2 | 3 | \pagenumbering{Roman} 4 | -------------------------------------------------------------------------------- /hack/down.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | docker-compose down 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /build 2 | /output-tex 3 | .idea 4 | /results/*.pdf 5 | __pycache__ 6 | -------------------------------------------------------------------------------- /hack/make.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | $(dirname "$0")/shell.sh make "$@" 4 | -------------------------------------------------------------------------------- /pandoc/filters/remove-date.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | jq -c 'del(.meta.date)' 4 | -------------------------------------------------------------------------------- /content/92-references.md: -------------------------------------------------------------------------------- 1 | \chapter*{References} 2 | \addcontentsline{toc}{chapter}{References} 3 | 4 | ::: {#refs} 5 | ::: 6 | -------------------------------------------------------------------------------- /assets/dhbw-cas.pdf: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:aa2862e0dd9d2878f7fce39df5bd6b7e53c9b8030a6bd504a077d77a149af06a 3 | size 38963 4 | -------------------------------------------------------------------------------- /assets/design-overview.pdf: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:caef35090dae057e11e01956709c0977919a85507b0aa64f17e4a8daa867c928 3 | size 45858 4 | -------------------------------------------------------------------------------- /assets/sample-website.png: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:d612643ef89d8650b4b2672d0764dd6894e87c022ed57e4b4666585d9a7d26d5 3 | size 98710 4 | -------------------------------------------------------------------------------- /assets/cluster-architecture.pdf: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:7088d57ccbbfbc71f2fd11545a7439492e02d498f537b3c1afe814f598589573 3 | size 16006 4 | -------------------------------------------------------------------------------- /assets/cluster-architecture.png: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:de6597803462efa89d2e350b766a51802bb6fce26dace31b3ae1c9f87d8d7f79 3 | size 87504 4 | -------------------------------------------------------------------------------- /assets/dashboard-sharding.png: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:7c838741795d7d2783d9cc4eed38faa922821ec57bddd49bec8874da52a86c08 3 | size 499139 4 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | assets/* filter=lfs diff=lfs merge=lfs -text 2 | pandoc/fonts/*.ttf filter=lfs diff=lfs merge=lfs -text 3 | paper-final.pdf filter=lfs diff=lfs merge=lfs -text 4 | -------------------------------------------------------------------------------- /assets/controller-components.jpeg: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:c2e110c1f504983ac516a6ecd963e75866a49bc2c9303d74b023c5bc7ee593e8 3 | size 111826 4 | -------------------------------------------------------------------------------- /assets/dashboard-webhosting.png: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:7927e7fd222aa405fd0b1c38ddfa771db588e4ebafa3ec78147acc6076b60a9e 3 | size 373270 4 | -------------------------------------------------------------------------------- /pandoc/fonts/JetBrainsMono-Bold.ttf: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:ce4dff4e5a334288c54295ec4e84a4a2640c47a7884aec91dfecdb41d789a5a0 3 | size 114240 4 | -------------------------------------------------------------------------------- /pandoc/fonts/JetBrainsMono-Thin.ttf: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:e06c642cc493e7ebcdf15d9cb0e409a4d71183628805d73340f80a23c47f1726 3 | size 114420 4 | -------------------------------------------------------------------------------- /assets/dashboard-controller-runtime.png: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:a1e30b25a21efa7fdb2b71e00f7bacc32daf03f827a97a5738f424df11641d7e 3 | size 883011 4 | -------------------------------------------------------------------------------- /pandoc/fonts/JetBrainsMono-ExtraBold.ttf: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:754ab9bd07b0434709bb50aa4bfd6ca890674d4df29efa498f680b6f1e73347c 3 | size 114232 4 | -------------------------------------------------------------------------------- /pandoc/fonts/JetBrainsMono-Italic.ttf: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:74cd0c6cfe2b3b5a449c5093dd63b5bb2384aea9815530461fecb9289a989008 3 | size 117108 4 | -------------------------------------------------------------------------------- /pandoc/fonts/JetBrainsMono-Light.ttf: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:b97fe495b4272436201ba54a20b89b2e6dc4a5a92151c228ea6262d443f8125f 3 | size 114444 4 | -------------------------------------------------------------------------------- /pandoc/fonts/JetBrainsMono-Medium.ttf: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:14708d08d033c34c55100773d6ed09727e82eaf4d6a1bb23be5540ecdb5b16a6 3 | size 114348 4 | -------------------------------------------------------------------------------- /pandoc/fonts/JetBrainsMono-Regular.ttf: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:4391a6a20152f6c1cd05524e81c594b1a556117e027332a1c7042e799b7777e8 3 | size 114320 4 | -------------------------------------------------------------------------------- /pandoc/fonts/JetBrainsMono-SemiBold.ttf: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:808c6df3acd8d1de35536619d429a095b082a1c07978650bcbd3dca44bda7c4d 3 | size 114304 4 | -------------------------------------------------------------------------------- /pandoc/fonts/JetBrainsMono-BoldItalic.ttf: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:e80bf940d06157eafab63706f688796271fc8b65837b021cc5df7feddaed8b77 3 | size 117108 4 | -------------------------------------------------------------------------------- /pandoc/fonts/JetBrainsMono-ExtraLight.ttf: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:a0c46b988ea868396dd0cf6c8137ee4eca68879a9b27e98983a975c640c7873d 3 | size 114508 4 | -------------------------------------------------------------------------------- /pandoc/fonts/JetBrainsMono-LightItalic.ttf: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:760e419914468d5382fb4b952cdda6f4d9aee1da7462fafb3a56c450335f1194 3 | size 117312 4 | -------------------------------------------------------------------------------- /pandoc/fonts/JetBrainsMono-MediumItalic.ttf: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:36df6194a23a05beffd27c5456c4cb5480bd0248eb18176e48103db42120183f 3 | size 117236 4 | -------------------------------------------------------------------------------- /pandoc/fonts/JetBrainsMono-SemiBoldItalic.ttf: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:ece694e49a277a7aa3a21d0f761ad42924edf4537c30e0c552f63d8fd3e451e8 3 | size 117176 4 | -------------------------------------------------------------------------------- /pandoc/fonts/JetBrainsMono-ThinItalic.ttf: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:833f749ff0c5cb6e96790e04e121b700bf19b49349063196b2ee17140a0a10ac 3 | size 117340 4 | -------------------------------------------------------------------------------- /pandoc/fonts/JetBrainsMono-ExtraBoldItalic.ttf: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:5c4e82122f42a27d685363a5212b31603047d7120ac6e220d34c57f6bf24e21d 3 | size 117156 4 | -------------------------------------------------------------------------------- /pandoc/fonts/JetBrainsMono-ExtraLightItalic.ttf: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:557cdf71deab3cbb4937327b2279c2412c03294a92b80bcae76f433b75973dd6 3 | size 117436 4 | -------------------------------------------------------------------------------- /pandoc/.latexmkrc: -------------------------------------------------------------------------------- 1 | $latex = 'latex -interaction=nonstopmode -shell-escape'; 2 | $pdflatex = 'pdflatex -interaction=nonstopmode -shell-escape'; 3 | $xelatex = 'xelatex -interaction=nonstopmode -shell-escape'; 4 | -------------------------------------------------------------------------------- /content/03-toc.md: -------------------------------------------------------------------------------- 1 | \begingroup 2 | \hypersetup{linkcolor=black} 3 | \tableofcontents 4 | \endgroup 5 | 6 | \newpage 7 | 8 | \cleardoublepage 9 | \setcounter{savepage}{\arabic{page}} 10 | \pagenumbering{arabic} 11 | -------------------------------------------------------------------------------- /pandoc/metadata.yaml: -------------------------------------------------------------------------------- 1 | title: Towards Horizontally Scalable Kubernetes Controllers 2 | author: Tim Ebert 3 | company: SAP SE / Schwarz IT KG 4 | email: tim.ebert@sap.com 5 | enrollment-number: 8559152 6 | date: June, 2022 7 | lang: en-US 8 | -------------------------------------------------------------------------------- /docker-compose.yaml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | 3 | services: 4 | paper: 5 | # temporarily use manually built arm64 image 6 | image: ghcr.io/timebertt/markdown-paper-setup/python:arm64 7 | command: 8 | - -c 9 | - "while true; do sleep 2; done" 10 | stdin_open: true 11 | tty: true 12 | cpu_count: 8 13 | volumes: 14 | - ".:/data" 15 | -------------------------------------------------------------------------------- /hack/up.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | docker-compose up -d 4 | 5 | $(dirname "$0")/make.sh install-python-requirements 6 | 7 | >&2 cat </dev/null)" == "running" ]; then 6 | >&2 echo "service paper is not running, please run \`./hack/up.sh\` and try again" 7 | exit 1 8 | fi 9 | 10 | docker exec -it "$container" "${@:-sh}" 11 | -------------------------------------------------------------------------------- /results/common.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | 3 | 4 | def read_data(filename): 5 | data = pd.read_csv( 6 | filename, 7 | parse_dates=['ts'], 8 | date_parser=lambda col: pd.to_datetime(col, utc=True, unit='s'), 9 | ) 10 | ts_min = data.ts.min() 11 | data.ts = (data.ts - ts_min).astype('timedelta64[s]') 12 | return data.pivot(index='ts', columns=data.columns.drop(labels=['ts', 'value']), values='value') 13 | -------------------------------------------------------------------------------- /content/91-lists.md: -------------------------------------------------------------------------------- 1 | \begingroup 2 | \hypersetup{linkcolor=black} 3 | 4 | 8 | 9 | \setlength{\parskip}{0em} 10 | 11 | 17 | 18 | \listoffigures 19 | \listoftables 20 | \listoflistings 21 | \addcontentsline{toc}{chapter}{List of Code} 22 | 23 | \endgroup 24 | 25 | \newpage 26 | -------------------------------------------------------------------------------- /pandoc/defaults-latex.yaml: -------------------------------------------------------------------------------- 1 | to: latex 2 | 3 | metadata: 4 | listings: true 5 | 6 | variables: 7 | documentclass: scrreprt 8 | 9 | classoption: 10 | - a4paper 11 | - fontsize=12pt 12 | - numbers=noenddot 13 | - chapterprefix=false 14 | - headings=optiontohead 15 | - svgnames 16 | - dvipsnames 17 | - listof=nochaptergap # no chapter gap in list of figures/tables 18 | # - oneside 19 | 20 | hyperrefoptions: 21 | - linktoc=all 22 | # - hidelinks 23 | 24 | listings: false 25 | pdf-engine: latexmk 26 | pdf-engine-opt: 27 | - "-xelatex" 28 | - "-r" 29 | - "pandoc/.latexmkrc" 30 | - "-g" 31 | - "-outdir=output-tex" 32 | 33 | include-in-header: 34 | - pandoc/includes/header.tex 35 | -------------------------------------------------------------------------------- /pandoc/filters/count-words.lua: -------------------------------------------------------------------------------- 1 | -- counts words in a document 2 | 3 | words = 0 4 | 5 | wordcount = { 6 | Str = function(el) 7 | -- we don't count a word if it's entirely punctuation: 8 | if el.text:match("%P") then 9 | words = words + 1 10 | end 11 | end, 12 | 13 | Code = function(el) 14 | _,n = el.text:gsub("%S+","") 15 | words = words + n 16 | end, 17 | 18 | CodeBlock = function(el) 19 | _,n = el.text:gsub("%S+","") 20 | words = words + n 21 | end 22 | } 23 | 24 | function Pandoc(el) 25 | -- skip metadata, just count body: 26 | pandoc.walk_block(pandoc.Div(el.blocks), wordcount) 27 | print(words .. " words in body") 28 | os.exit(0) 29 | end 30 | -------------------------------------------------------------------------------- /.github/workflows/build.yaml: -------------------------------------------------------------------------------- 1 | name: Build 2 | 3 | on: 4 | release: 5 | types: 6 | - published 7 | push: 8 | branches: 9 | - master 10 | tags: 11 | - v* 12 | 13 | jobs: 14 | build: 15 | runs-on: ubuntu-latest 16 | steps: 17 | - uses: actions/checkout@v3 18 | with: 19 | lfs: true 20 | - run: git lfs checkout 21 | - uses: docker://ghcr.io/timebertt/markdown-paper-setup/python:sha-26f1955 22 | id: build 23 | with: 24 | entrypoint: /bin/sh 25 | args: -c "make install-python-requirements pdf" 26 | - uses: actions/upload-artifact@v3 27 | with: 28 | name: paper 29 | path: build/paper.pdf 30 | if-no-files-found: error 31 | -------------------------------------------------------------------------------- /content/90-appendix.md: -------------------------------------------------------------------------------- 1 | \newpage 2 | 3 | \cleardoublepage 4 | \pagenumbering{Roman} 5 | \setcounter{page}{\thesavepage} 6 | 7 | \appendix 8 | 9 | \chapter*{Appendix} 10 | \addcontentsline{toc}{chapter}{Appendix} 11 | 12 | The implementation belonging to this thesis (including webhosting operator, monitoring setup, experiment tools, etc.) is available at: 13 | [https://github.com/timebertt/kubernetes-controller-sharding/tree/v1.0](https://github.com/timebertt/kubernetes-controller-sharding/tree/v1.0) 14 | 15 | The sharding implementation in the controller-runtime library is available at: 16 | [https://github.com/timebertt/controller-runtime/tree/v0.12.3-sharding-1](https://github.com/timebertt/controller-runtime/tree/v0.12.3-sharding-1) 17 | -------------------------------------------------------------------------------- /results/base-websites-plot.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import matplotlib.pyplot as plt 4 | 5 | from common import * 6 | 7 | plt.figure(figsize=(10, 6)) 8 | 9 | ax1 = plt.subplot(121) 10 | data = read_data('base/sharded-websites_per_project.csv') 11 | data.plot( 12 | title='Websites per Project (stacked)', 13 | legend=False, 14 | grid=True, 15 | xlabel='Time in seconds', 16 | ylabel='Website count', 17 | xlim=[0, data.index.max()], 18 | kind='area', 19 | ax=ax1, 20 | ) 21 | plt.ylim(bottom=0) 22 | 23 | ax2 = plt.subplot(122) 24 | data = read_data('base/sharded-websites_per_shard.csv') 25 | data.plot( 26 | title='Websites per Shard', 27 | legend=False, 28 | grid=True, 29 | xlabel='Time in seconds', 30 | xlim=[0, data.index.max()], 31 | ax=ax2, 32 | ) 33 | plt.ylim(bottom=0) 34 | 35 | plt.savefig('base-websites.pdf', bbox_inches='tight') 36 | -------------------------------------------------------------------------------- /.run/Plots.run.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 17 | -------------------------------------------------------------------------------- /results/base-cpu-plot.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import matplotlib.pyplot as plt 4 | 5 | from common import * 6 | 7 | plt.figure(figsize=(10, 5)) 8 | 9 | ax1 = plt.subplot(121) 10 | data = read_data('base/sharded-cpu.csv') 11 | data.plot( 12 | title='Sharded', 13 | legend=False, 14 | grid=True, 15 | xlabel='Time in seconds', 16 | ylabel='CPU usage in cores', 17 | xlim=[0, data.index.max()], 18 | ax=ax1, 19 | ) 20 | 21 | ax2 = plt.subplot(122, sharex=ax1, sharey=ax1) 22 | data = read_data('base/singleton-cpu.csv') 23 | data.plot( 24 | title='Singleton', 25 | legend=False, 26 | grid=True, 27 | xlabel='Time in seconds', 28 | ylabel='CPU usage in cores', 29 | xlim=[0, data.index.max()], 30 | ax=ax2, 31 | ) 32 | 33 | plt.ylim(bottom=0) 34 | # plt.suptitle('CPU Usage per Pod', weight='bold') 35 | plt.savefig('base-cpu.pdf', bbox_inches='tight') 36 | -------------------------------------------------------------------------------- /.run/Clean Plots.run.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 17 | -------------------------------------------------------------------------------- /.run/Build.run.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 17 | -------------------------------------------------------------------------------- /.run/Clean.run.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 17 | -------------------------------------------------------------------------------- /content/01-affirmation.md: -------------------------------------------------------------------------------- 1 | \chapter*{Erklärung} 2 | 3 | Hiermit erkläre ich, dass ich die vorliegende \TypeOfDocument\ selbstständig und ohne fremde Hilfe verfasst und keine anderen als die angegebenen Quellen und Hilfsmittel benutzt habe. 4 | Aus fremden Quellen direkt oder indirekt übernommene Gedanken habe ich als solche kenntlich gemacht. 5 | Die Arbeit habe ich bisher keinem anderen Prüfungsamt in gleicher oder vergleichbarer Form vorgelegt. 6 | Sie wurde bisher auch nicht veröffentlicht. 7 | 8 | Ich erkläre weiterhin, dass ich ein unverschlüsseltes digitales Textdokument der Arbeit 9 | (in einem der Formate .doc, .docx, .odt, .pdf, .rtf) in Moodle hochgeladen habe. Alle digital eingereichten Exemplare entsprechen in Inhalt und Wortlaut ausnahmslos der gedruckten Ausfertigung. 10 | 11 | \vspace{2.5cm} 12 | 13 | \begin{tabularx}{\textwidth}{XX} 14 | \hrulefill & \hrulefill \\ 15 | Ort, Datum & Unterschrift \\ 16 | \end{tabularx} 17 | 18 | \newpage 19 | -------------------------------------------------------------------------------- /results/base-memory-plot.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import matplotlib.pyplot as plt 4 | 5 | from common import * 6 | 7 | plt.figure(figsize=(10, 5)) 8 | 9 | ax1 = plt.subplot(121) 10 | data = read_data('base/sharded-memory.csv') 11 | data = data / (2 ** 20) 12 | data.plot( 13 | title='Sharded', 14 | legend=False, 15 | grid=True, 16 | xlabel='Time in seconds', 17 | ylabel='Memory usage in MiB', 18 | xlim=[0, data.index.max()], 19 | ax=ax1, 20 | ) 21 | 22 | ax2 = plt.subplot(122, sharex=ax1, sharey=ax1) 23 | data = read_data('base/singleton-memory.csv') 24 | data = data / (2 ** 20) 25 | data.plot( 26 | title='Singleton', 27 | legend=False, 28 | grid=True, 29 | xlabel='Time in seconds', 30 | ylabel='Memory usage in MiB', 31 | xlim=[0, data.index.max()], 32 | ax=ax2, 33 | ) 34 | 35 | plt.ylim(bottom=0) 36 | # plt.suptitle('Memory Usage per Pod', weight='bold') 37 | plt.savefig('base-memory.pdf', bbox_inches='tight') 38 | -------------------------------------------------------------------------------- /pandoc/defaults.yaml: -------------------------------------------------------------------------------- 1 | from: markdown+link_attributes+native_divs+raw_tex+tex_math_dollars+inline_code_attributes+grid_tables+fenced_code_attributes 2 | 3 | metadata: 4 | link-citations: true 5 | link-bibliography: true 6 | bibliography: content/bibliography.bib 7 | csl: pandoc/ieee.csl 8 | 9 | # pandoc-crossref options 10 | chapters: true 11 | autoSectionLabels: true 12 | # figPrefix: "Abb." 13 | # tblPrefix: "Tab." 14 | # eqnPrefix: "Formel" 15 | lstPrefix: code 16 | 17 | codeBlockCaptions: true 18 | 19 | highlight-style: pandoc/highlight-style.theme 20 | 21 | variables: 22 | linkcolor: Maroon 23 | filecolor: Black 24 | citecolor: Black 25 | urlcolor: MediumBlue 26 | toccolor: Black 27 | 28 | number-sections: true 29 | toc: false # insert TOC manually after abstract 30 | top-level-division: chapter 31 | resource-path: 32 | - "." 33 | - "assets" 34 | - "results" 35 | - "pandoc/fonts" 36 | 37 | eol: lf 38 | 39 | filters: 40 | - ./pandoc/filters/remove-date.sh 41 | - pandoc-crossref 42 | - ./pandoc/filters/pandoc-minted.py 43 | - ./pandoc/filters/pretty-tables.py 44 | - citeproc 45 | 46 | -------------------------------------------------------------------------------- /results/base.yaml: -------------------------------------------------------------------------------- 1 | # scenario: base 2 | # measure -o results/base results/base.yaml --output-prefix sharded- --start 2022-10-24T09:30:49+02:00 --end 2022-10-24T09:43:49+02:00 3 | # measure -o results/base results/base.yaml --output-prefix singleton- --start 2022-10-24T08:24:27+02:00 --end 2022-10-24T08:37:27+02:00 4 | queries: 5 | - name: websites_per_shard 6 | query: sum(kube_website_shard{shard!=""} or 0*sum(kube_shard_info{app="webhosting-operator"}) by (shard)) by (shard) 7 | optional: true 8 | - name: websites_per_project 9 | query: sum(kube_website_info{} or 0*sum(kube_namespace_status_phase{namespace=~"project-.+"}) by (namespace)) by (namespace) 10 | - name: websites_per_theme 11 | query: sum(kube_website_info{} or 0*sum(kube_theme_info{}) by (theme)) by (theme) 12 | - name: cpu 13 | query: sum(rate(container_cpu_usage_seconds_total{namespace="webhosting-system", container="manager"}[2m])) by (pod) 14 | - name: memory 15 | query: sum(container_memory_rss{namespace="webhosting-system", container="manager"}) by (pod) 16 | - name: network_receive 17 | query: sum(irate(container_network_receive_bytes_total{namespace="webhosting-system", pod=~"webhosting-operator-.+"}[2m])) by (pod) 18 | - name: network_transmit 19 | query: sum(irate(container_network_transmit_bytes_total{namespace="webhosting-system", pod=~"webhosting-operator-.+"}[2m])) by (pod) 20 | -------------------------------------------------------------------------------- /draw/cluster-architecture.drawio: -------------------------------------------------------------------------------- 1 | 3Vpdc6IwFP01PuoQAoiPre1uZ7rd6UwfuvsY4Sq0kbgxVN1fv4kEEIL9WKVUbR/IzRc5596Tm2gPj+fr75wsojsWAu3ZVrju4auebY9su6f+rXCTGYbIygwzHoeZCZWGh/gvaGPeLI1DWFYaCsaoiBdVY8CSBAJRsRHO2arabMpoddYFmYFheAgINa2PcSiizOq7Vmm/gXgW5TMjS9fMSd5YG5YRCdlqx4Sve3jMGRPZ03w9Bqqwy3HJ+n3bU1u8GIdEvKfD5Y0b9v9c3Y28u6fF+PbiCf3y+zk9L4SmesW36QR4AkKibltjmi4FcL0Csclh4SxNQlAjWz18uYpiAQ8LEqjalfQDaYvEnMoSko+ExrNEPgfyTeVY+PIFuIglxhe6QjDVY8oSoR0AuXlZT6mGmcaUjhllfPsKOCTgTwNpXwrOnmGnxgt8mExljV6bnA3We1FDBRfSh4HNQfCNbKI7eJo97b6FX65KZyhcOtpxBCf3A6IdcFYMXXIkHzRNH6HMMigbDAZfgKHDwXbraPsm2mjUgLbttoW2XGcd7UfGn1VIWD+l2J0H7k4Vd9zg5Z+NuwH7WC6aM9XxnpLkPID3a/7umrgXQrKLO3ZaU5ehASyEckfURcZFxGYsIfS6tF5WoS/b/GAKxS3gTyDERkNJUsGqdMA6Fr92nn+roQauLl2t9cjbwiYvSHfYqE59a2BZTm7RXf1hbih7b0uV7vfAY4mb8oCtsc53hoUC4HWCJV4s5QG8AqyjkxfCZyBeaec3OwwHSkT8Un2Po7PvGFH3LLOBfpCFHgXen5NEpjwHJgS1nXw6ndpB404eehPP9fYSc1DsFZmaDj7HaRA9t3Frbyv2RqcZe3Y19oYeeiP26mw2xuIRY887hdjzmmNvGUQQpvQ8Q27UdcgNDdBBBOFxofYDaIZ64ruOu3/bOQxq/HZK15TRtSdubifilgsVqmqU+0GFer8YvSkywwNFRne9Z7GcueC7IDPnG9eIzERS9yq5vOCcbHaaLVSD5cfnKV0jG7F0lGKN/+87frM0kkW8BP5ydGkEb480DkcTq6V4reNqN0mj13QEG7V2BDMvGs7w6GvX/LkpC/zcoy+2u0wD81zvQ2kgMk9gDrLf0NjPPoGh9yp0p2kgMm8+lNhREGeQABoXTZ0ngMg24FanXRInW53jaSKkfx4Xelf9aYB37Nmn8UJ7+/kcSrrPySUGJ6J/bYoVbiVNdNwa216NxUwcjTTxWJkcws2p3IKz9eYMBa77SyVk3ukd/HXRSQma2zkFGBlwn+K93gh/tYzu3WfuTjM628zozjsC69d8bUagLJY/och2qfJ3KPj6Hw== -------------------------------------------------------------------------------- /results/base-network-plot.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import matplotlib.pyplot as plt 4 | 5 | from common import * 6 | 7 | plt.figure(figsize=(10, 8)) 8 | 9 | ax1 = plt.subplot(221) 10 | data = read_data('base/sharded-network_receive.csv') 11 | data = data / (2 ** 20) 12 | data.plot( 13 | title='Sharded: Receive Bandwidth', 14 | legend=False, 15 | grid=True, 16 | xlabel='Time in seconds', 17 | ylabel='Receive Bandwidth in MiB/s', 18 | xlim=[0, data.index.max()], 19 | ax=ax1, 20 | ) 21 | 22 | ax2 = plt.subplot(222, sharex=ax1, sharey=ax1) 23 | data = read_data('base/singleton-network_receive.csv') 24 | data = data / (2 ** 20) 25 | data.plot( 26 | title='Singleton: Receive Bandwidth', 27 | legend=False, 28 | grid=True, 29 | xlabel='Time in seconds', 30 | ylabel='Receive Bandwidth in MiB/s', 31 | xlim=[0, data.index.max()], 32 | ax=ax2, 33 | ) 34 | 35 | plt.ylim(bottom=0) 36 | 37 | ax3 = plt.subplot(223) 38 | data = read_data('base/sharded-network_transmit.csv') 39 | data = data / (2 ** 20) 40 | data.plot( 41 | title='Sharded: Transmit Bandwidth', 42 | legend=False, 43 | grid=True, 44 | xlabel='Time in seconds', 45 | ylabel='Transmit Bandwidth in MiB/s', 46 | xlim=[0, data.index.max()], 47 | ax=ax3, 48 | ) 49 | 50 | ax4 = plt.subplot(224, sharex=ax3, sharey=ax3) 51 | data = read_data('base/singleton-network_transmit.csv') 52 | data = data / (2 ** 20) 53 | data.plot( 54 | title='Singleton: Transmit Bandwidth', 55 | legend=False, 56 | grid=True, 57 | xlabel='Time in seconds', 58 | ylabel='Transmit Bandwidth in MiB/s', 59 | xlim=[0, data.index.max()], 60 | ax=ax4, 61 | ) 62 | 63 | plt.ylim(bottom=0) 64 | # plt.suptitle('Network Bandwidth per Pod', weight='bold') 65 | plt.savefig('base-network.pdf', bbox_inches='tight') 66 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | BUILDDIR := build 2 | MD_IN := content/*.md 3 | HTML := $(BUILDDIR)/paper.html 4 | PDF := $(BUILDDIR)/paper.pdf 5 | METADATA := pandoc/metadata.yaml 6 | 7 | PLOTS_DIR := results 8 | PLOT_CSV_FILES := $(wildcard $(PLOTS_DIR)/*.csv $(PLOTS_DIR)/*/*.csv) 9 | PLOT_PY_FILES := $(wildcard $(PLOTS_DIR)/*-plot.py) 10 | PLOT_COMMON_PY_FILES := $(filter-out $(PLOT_PY_FILES),$(wildcard $(PLOTS_DIR)/*.py)) 11 | PLOT_PDF_FILES := $(patsubst $(PLOTS_DIR)/%-plot.py,$(PLOTS_DIR)/%.pdf,$(PLOT_PY_FILES)) 12 | 13 | all: html pdf 14 | 15 | $(BUILDDIR): 16 | @mkdir $(BUILDDIR) -p 17 | 18 | pdf: $(PDF) 19 | .PHONY: $(PDF) 20 | $(PDF): $(BUILDDIR) $(MD_IN) $(PLOT_PDF_FILES) 21 | @echo "> Building PDF" 22 | @pandoc $(MD_IN) \ 23 | --fail-if-warnings \ 24 | --defaults "pandoc/defaults.yaml" \ 25 | --defaults "pandoc/defaults-latex.yaml" \ 26 | --metadata-file $(METADATA) \ 27 | --output=$(PDF) 28 | 29 | open: open-pdf 30 | open-pdf: 31 | @open $(PDF) 32 | 33 | html: $(HTML) 34 | .PHONY: $(HTML) 35 | $(HTML): $(BUILDDIR) $(MD_IN) $(PLOT_PDF_FILES) 36 | @echo "> Building HTML" 37 | @pandoc $(MD_IN) \ 38 | --fail-if-warnings \ 39 | --defaults "pandoc/defaults.yaml" \ 40 | --metadata-file $(METADATA) \ 41 | --to=html5 \ 42 | --output=$(HTML) \ 43 | --self-contained 44 | 45 | open-html: 46 | @open $(HTML) 47 | 48 | plots: $(PLOT_PDF_FILES) 49 | 50 | $(PLOTS_DIR)/%.pdf: $(PLOTS_DIR)/%-plot.py $(PLOT_COMMON_PY_FILES) $(PLOT_CSV_FILES) 51 | @echo "> Plotting $(@F)" 52 | @cd $( 0: 23 | language = classes[0] 24 | 25 | caption = None 26 | captionA = [a[1] for a in attributes if a[0] == 'caption'] 27 | if len(captionA) > 0: 28 | caption = captionA[0] 29 | 30 | attributes = [a for a in attributes if a[0] != 'caption'] 31 | attributes = ', '.join('='.join(x) for x in attributes) 32 | 33 | return {'label': label, 'contents': contents, 'language': language, 34 | 'attributes': attributes, 'caption': caption} 35 | 36 | 37 | def process_code(key, value, format, meta): 38 | ''' Use minted for code in LaTeX. 39 | 40 | Args: 41 | key type of pandoc object 42 | value contents of pandoc object 43 | format target output format 44 | meta document metadata 45 | ''' 46 | if format != 'latex': 47 | return 48 | 49 | # Determine what kind of code object this is. 50 | if key == 'CodeBlock': 51 | template = Template(''' 52 | \\begin{listing}[ht] 53 | \\begin{minted}[$attributes]{$language} 54 | $contents 55 | \\end{minted} 56 | \\caption{$caption} 57 | \\label{$label} 58 | \\end{listing} 59 | ''') 60 | Element = RawBlock 61 | elif key == 'Code': 62 | template = Template('\\mintinline[$attributes]{$language}{$contents}') 63 | Element = RawInline 64 | else: 65 | return 66 | 67 | # print(value, file=sys.stderr) 68 | code = unpack_code(value, 'text') 69 | # print(code, file=sys.stderr) 70 | 71 | return [Element(format, template.substitute(code))] 72 | 73 | 74 | if __name__ == '__main__': 75 | toJSONFilter(process_code) 76 | -------------------------------------------------------------------------------- /content/02-abstract.md: -------------------------------------------------------------------------------- 1 | \chapter*{Abstract} 2 | 3 | Controllers play an essential role in every Kubernetes cluster by realizing the desired state specified declaratively in API objects – a process referred to as reconciliation. 4 | To prevent uncoordinated and conflicting actions of concurrent reconciliations, controllers use a leader election mechanism to determine a single active controller instance. 5 | Because of this, reconciliations cannot be distributed among multiple controller instances which effectively limits the system's capacity and throughput by the available machine sizes and network bandwidth of the underlying infrastructure. 6 | I.e., leader election prevents Kubernetes controllers from being horizontally scalable. 7 | 8 | This thesis presents the first steps towards horizontally scalable Kubernetes controllers by introducing a design for distributing the reconciliation of API objects across multiple controller instances, i.e., sharding for Kubernetes controllers. 9 | It shows that proven sharding mechanisms used in distributed databases can be applied to the problem space of Kubernetes controllers as well to overcome the mentioned scalability limitation. 10 | The proposed design includes lease-based membership and failure detection as well as consistent hashing for partitioning. 11 | Kubernetes API machinery primitives – namely, labels and label selectors – facilitate coordination between controller instances and prevent concurrent reconciliations of individual objects by multiple instances. 12 | 13 | For demonstration and evaluation, the sharding design is implemented based on the controller-runtime library and used in an example operator. 14 | Systematic load test experiments show that the sharding implementation achieves a good distribution of object responsibility and resource usage across individual shard instances. 15 | However, the experiment results also point out that one of the controller instances always has a high resource footprint in comparison to the other shards which limits the scalability of the system. 16 | Nevertheless, future work can perform optimizations of the presented design to overcome the discovered flaw. 17 | 18 | The presented design and implementation allow scaling Kubernetes controllers horizontally providing potential for a wide range of applications in the Kubernetes ecosystem. 19 | 20 | \newpage 21 | -------------------------------------------------------------------------------- /content/10-motivation.md: -------------------------------------------------------------------------------- 1 | # Motivation 2 | 3 | In Kubernetes, the desired and actual state of the distributed system are stored centrally in etcd [@etcddocs] and exposed by the Kubernetes API server. 4 | The logic for turning the desired state into the actual state of the system (reconciliation) is implemented in dedicated components called controllers. 5 | Storing the desired and actual state is the API server's responsibility and reconciling both is the task of controllers. 6 | Because these tasks are decoupled Kubernetes is an asynchronous and eventually consistent [@brewer2000towards; @vogels2008eventually] system. [@k8sdocs] 7 | 8 | Kubernetes controllers as such are stateless components, they persist the state they work on in external systems – in etcd via the API server. 9 | I.e., the system doesn't lose any important state if controllers fail or restart. 10 | However, Kubernetes controllers typically use a leader election mechanism to determine a single active controller instance (leader). 11 | When deploying multiple instances of the same controller, there will only be one active instance at any given time, other instances will be on standby. 12 | This is done to prevent controllers from performing uncoordinated and conflicting actions. 13 | 14 | If the current leader stops or loses leadership (e.g., during a rolling update or network partition) another instance takes over leadership and becomes the active instance. 15 | Such a setup can be described as an "active-passive high-availability (HA) setup" [@ahluwalia2006high]. 16 | It minimizes controller downtime and facilitates fast failovers. 17 | However, it cannot be considered horizontal scaling because work is not distributed across multiple instances [@bondi2000characteristics; @jogalekar2000evaluating]. 18 | Because of this, Kubernetes controllers are not horizontally scalable. 19 | 20 | This restriction imposes scalability limitations for Kubernetes controllers. 21 | I.e., the rate of reconciliations, amount of objects, etc. are limited by the available machine size and network bandwidth of the underlying infrastructure. 22 | In contrast to usual stateless applications, one cannot increase the capacity and throughput of the system by adding more instances (horizontal scaling) but only by using instances with more resources (vertical scaling). 23 | 24 | This thesis explores approaches for distributing the reconciliation of Kubernetes objects across multiple controller instances. 25 | It attempts to lift the restriction of having only one active instance per controller. 26 | For this, mechanisms are required for determining which instance is responsible for which object to prevent conflicting actions. 27 | The thesis evaluates how proven mechanisms for sharding and partitioning from the field of distributed databases [@abadi2009data; @agrawal2004integrating] can be applied to this problem. 28 | -------------------------------------------------------------------------------- /pandoc/filters/pretty-tables.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | """ 4 | Pandoc filter to render latex tables. The default pandoc tables don't use vertical lines 5 | which looks weird, so generate the tables with this filter as desired. 6 | """ 7 | 8 | from pandocfilters import RawBlock, RawInline, Para, toJSONFilter 9 | 10 | 11 | def latex(s): 12 | return RawBlock('latex', s) 13 | 14 | 15 | def inlatex(s): 16 | return RawInline('latex', s) 17 | 18 | 19 | def tbl_caption(s): 20 | return Para([inlatex(r'\caption{')] + s + [inlatex(r'}')]) 21 | 22 | 23 | def tbl_alignment(s): 24 | aligns = { 25 | 'AlignDefault': r'\raggedright', 26 | 'AlignLeft': r'\raggedright', 27 | 'AlignCenter': r'\centering', 28 | 'AlignRight': r'\raggedleft', 29 | } 30 | 31 | cols = len(s) 32 | result = [] 33 | 34 | for col in s: 35 | align = aligns[col[0]['t']] 36 | width = col[1]['c'] 37 | result.append(r'>{%s\arraybackslash}p{(\columnwidth - %d\tabcolsep) * \real{%.4f}}' % (align, cols*2, width)) 38 | 39 | return result 40 | 41 | 42 | def tbl_headers(s): 43 | result = [inlatex(r'\hline' '\n')] 44 | 45 | for header in s[1][0][1]: 46 | result.append(inlatex(r'\textbf{')) 47 | result.extend(header[4][0]['c']) 48 | result.append(inlatex(r'}')) 49 | result.append(inlatex(' & ')) 50 | result.pop() 51 | result.append(inlatex(r' \\' '\n' r'\hline\hline')) 52 | 53 | return Para(result) 54 | 55 | 56 | def tbl_contents(s): 57 | result = [] 58 | 59 | for row in s[0][3]: 60 | for col in row[1]: 61 | result.extend(col[4][0]['c']) 62 | result.append(inlatex(' & ')) 63 | result.pop() 64 | result.append(inlatex(r' \\' '\n')) 65 | result.append(inlatex(r'\hline' '\n')) 66 | 67 | return Para(result) 68 | 69 | 70 | def process_table(key, value, format, meta): 71 | if format != 'latex': 72 | return 73 | 74 | if key == 'Table': 75 | caption = value[1] 76 | alignment = value[2] 77 | headers = value[3] 78 | contents = value[4] 79 | 80 | # there are `Table` elements in meta, skip them by looking at the caption value 81 | if len(caption[1]) == 0: 82 | return 83 | 84 | latex_alignment = '\n |' + '\n |'.join(tbl_alignment(alignment)) + '|' 85 | return [ 86 | latex(r'\begin{table}[h]'), 87 | latex(r'\centering'), 88 | latex(r'\begin{tabular}{@{} %s @{}}' % latex_alignment), 89 | tbl_headers(headers), 90 | tbl_contents(contents), 91 | latex(r'\end{tabular}'), 92 | tbl_caption(caption[1][0]['c']), 93 | latex(r'\end{table}') 94 | ] 95 | 96 | 97 | if __name__ == '__main__': 98 | toJSONFilter(process_table) 99 | -------------------------------------------------------------------------------- /draw/design-overview.drawio: -------------------------------------------------------------------------------- 1 | 7V1bc5s4FP41ntl9cIY7+LFOmmZ3stvOZmYvfekIEDYtRingOOmvXwnEVQJjc3Wa7uzUCEmAzne+c9GBLuTr3fOHADxu/0A29BaSYD8v5JuFJImioOG/SMtL0mKoUtKwCVybdsobHtwfkDYKtHXv2jAsdYwQ8iL3sdxoId+HVlRqA0GADuVuDvLKV30EG8g0PFjAY1v/ce1omz6XtspP3EF3s6WXNiQ9ObEDaWf6JOEW2OhQaJLfL+TrAKEo+bV7voYeWbx0XZJxtzVnsxsLoB+1GbAT1LuNHy4/3/+9Vr6KS8u3vixVmd5c9JI+MbTxAtBDFERbtEE+8N7nresA7X0bkmkFfJT3uUfoETeKuPErjKIXKk2wjxBu2kY7j57Fdxy8/EvGX6np4X/FczfPdPLk6IUeOciPbsHO9UjDHfSeYORaAJ9g1yLFCgg2MGpaACod8tSFkXQJP0C0g/gOcIcAeiByn8qwABRdm6xfNvQTcvG9SALVBHWlJkOoHmiGUJ4iRPvAgnRULsZ3QQBeCt0eSYfw9OvQ6fCPZMb0qPCMeVOMnBNQRFfjCXh7uj6uH0bAt+ASMPgqo+ewdSP48AgscvaAOaSMFOC5Gx//trBgYYAbnmBAZO69oyciArl1iCdw/c09dIisCaYc1/OukYeC+KKyo5L/KIQK7ckfMkMUoG+wcEaL/zRhi9wKfG4EDT2byprKRFopyfEhZxRZpn22BTKRZKEeZwWRniExZQq9L+nvNQa8i2UqCX/CQ4kUipRQYIgaUoDPbpRwiaTSw4xL8O98FDlIB9UKNVHDhpWTWxKLKA9CLKfygSjJJezpyhh8sGIIwYPAjoUNPWymXeQvJM3D67c2caO2Ib9+Adb3vRtA+1cuNu+BiX2L09hh59p2Al0Yuj+AGc9H5E+XDU+urhfqzWm2pVGjGFbIfBN6+ZL557GFcCUYxqoktRR1HY2QKHFnTSdAjhPCaFHllh4AIV2um1GlqfNdDXEkV0MWx3E16q5ze17/YahIStUyp6IDiKwtDC+RZFI96oNkRE3TS/Loh2PSESnHqPpYJCMzksYhVxBbHY6pIY/IGpqTnNOKj2kDaDgW15e0DGg6BSx5saO69gjaPqHQjQ0iB2L3lQ4Z1Nq6wT04r6uy8yrLrPMq8pxXsUo7/Tmv6iTWJPMz27uZdRYotzl9WqCjvqsymaXqJm+D0ewdwKSD/79EEk/h253El5jFy2wr9uMnliYVK3o8HIOzttpBaIkXBROq5xEVeKPrY3Sd5Qya+Fofl6+1Kfi6OdnAJXOhLZkLSjmguBKOxRTx0ScYuHhJCWZGykIYfRP5eUkItez5S1pzpCA19x8oaaEz7JM6jgRK+IwDqEy073uStV/n5J81pR5mHGQspGvW94xZJG3Gd5pMnpyb3Jwd9zUaNbwfk6YrRln8vRg1vWIqJWMss6YywIrNGsBRKAuP+pY6FJZlxiAR3z2wQQSWyMcibom7ToZVhYat8AyrIZlynFO/RMMqChXDqnMMq8gzrINl8bVJgRVL7SHO5xIBv+NeAOyIQHwzfIyPhTgkjzsLxT2i4VHpOFCzuO6era9MQbhQVEpKC1Sqo6KSjdYyZJTEmeKJiHzpUAtEgJHBpgaHpZMJzCrQvEWogCkzB5SAzK8Yr2zyb2DC4yOiD/lLZVeJI35N44lfGkr8nL0foidhR4YYlwr48mp0lSrsMYBsRZEVblZeUhSuNlgkJzTpNs+qJG33EISwluV5itpsq5I2H+xgnSnJR26RR3LAE5gcKGJm0Hk4W2m6DLTTne8zsgH6VRlEKqf4IIvPxgERWy6SSgObSJ9rI2q8jyKp4HtJhrMQajVhOsB2n04cUXvXJrC+bWIELa1E+AkIscMAvLop2/CheeShcXP8GIP4UY5UQ56aqan1xm5k8lSU2ZGnNGvcn3APLVFq1d5in3g0LMjHo2moijobY65ps8Mju2HKGPOq4T7bQudJ/GVaCvRmqIt0JVT8+NTVKpppY0x4ZHXLM9hlbZ2YH7icuLcdVhokTZyXr1bkHMvLy/IEeXmJdRfrk+sBxERjuRiLU2fSz6nuSaOrPqp7FLFc+DnEzvCyOsGAxT3s7kzRG+5iDF5HjbkilROBvCJzibfvO1yRuTrJvm/HOp1EVOdV6hR1nrvha4NwGz+b2MnKyEZLM9N/EXonQMhsari+RI8ot22+vP4avXagOIEJWlSAjFsAos3IlWwmggFcQnnVUln7fxWtm7Kyify3aqxTdVEVj+viyNVYyoys8ghxXX/Vs60VOU1wTxzdVV8FPRbdqVNUXSmsX/9Kozultxop4UqQdTpbx3huWeGnZTUYGLDYl43rW1SvnFmrUlOG10f1ijlGhrLFrstFmsdK0DqD6hWF3aUpbmi8JRp0YX6JBn0+Lk3LREPmw7R7yb3W+PSUUVCme629m7ae8tLfW0bhXJVvEcWMHMQY89H4y9mcav3+nzSLAMbQTwtg9Ob+AwUwbJrktQYwafa2hwBGk42OG1JUxquxwhVN5Mpszj5Gl5eOR2YctfeUSTfz8vYmag9eQ5svB4zsNpz1Ktb8cg7WKDmH45V1lwhLo817XOPmHNKrjVKJxyaujpTgjZLgmkEJnirOr1ZeZZOkIyDDaomMUWhoBsjQVzNEBltLwfnG3qyjjDo5nBxPKJXaanW4VW/1ZcOfY9V1ZaxV19ic/IV8AKi3xa5u5g642GxK9Wdb7GriaUA+meQ7vH1mFSbIY7bdq+g/qXBWGrP69Ztj38msoq/v72R+/u2j/vzl480+OvwOt3/dqdb1Z84nvGkWc3yNZ4HRqDq9fHPGMAxqYzrmI9OPvXRJSOLD/J8FSLrn/7iC/P5/ -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Towards Horizontally Scalable Kubernetes Controllers 2 | 3 | Read the full paper [here](https://github.com/timebertt/thesis-controller-sharding/releases/tag/v1.0)! 4 | 5 | ## About 6 | 7 | This is a study thesis (study project / half-time thesis) part of my master's studies in Computer Science at the [DHBW Center for Advanced Studies](https://www.cas.dhbw.de/) (CAS). 8 | 9 | You can find the implementation that was done as part of this thesis in the repository [kubernetes-controller-sharding](https://github.com/timebertt/kubernetes-controller-sharding/tree/v1.0). 10 | 11 | I continued the project and research in my Master's thesis "Horizontally Scalable Kubernetes Controllers". 12 | You can find the Master's thesis in the [masters-thesis-controller-sharding](https://github.com/timebertt/masters-thesis-controller-sharding) repository. 13 | 14 | ## Abstract 15 | 16 | Controllers play an essential role in every Kubernetes cluster by realizing the desired state specified declaratively in API objects – a process referred to as reconciliation. 17 | To prevent uncoordinated and conflicting actions of concurrent reconciliations, controllers use a leader election mechanism to determine a single active controller instance. 18 | Because of this, reconciliations cannot be distributed among multiple controller instances which effectively limits the system's capacity and throughput by the available machine sizes and network bandwidth of the underlying infrastructure. 19 | I.e., leader election prevents Kubernetes controllers from being horizontally scalable. 20 | 21 | This thesis presents the first steps towards horizontally scalable Kubernetes controllers by introducing a design for distributing the reconciliation of API objects across multiple controller instances, i.e., sharding for Kubernetes controllers. 22 | It shows that proven sharding mechanisms used in distributed databases can be applied to the problem space of Kubernetes controllers as well to overcome the mentioned scalability limitation. 23 | The proposed design includes lease-based membership and failure detection as well as consistent hashing for partitioning. 24 | Kubernetes API machinery primitives – namely, labels and label selectors – facilitate coordination between controller instances and prevent concurrent reconciliations of individual objects by multiple instances. 25 | 26 | For demonstration and evaluation, the sharding design is implemented based on the controller-runtime library and used in an example operator. 27 | Systematic load test experiments show that the sharding implementation achieves a good distribution of object responsibility and resource usage across individual shard instances. 28 | However, the experiment results also point out that one of the controller instances always has a high resource footprint in comparison to the other shards which limits the scalability of the system. 29 | Nevertheless, future work can perform optimizations of the presented design to overcome the discovered flaw. 30 | 31 | The presented design and implementation allow scaling Kubernetes controllers horizontally providing potential for a wide range of applications in the Kubernetes ecosystem. 32 | -------------------------------------------------------------------------------- /results/base/singleton-memory.csv: -------------------------------------------------------------------------------- 1 | ts,value,pod 2 | 1666592667,24592384,webhosting-operator-6b5d9d485f-r99lx 3 | 1666592682,24592384,webhosting-operator-6b5d9d485f-r99lx 4 | 1666592697,24727552,webhosting-operator-6b5d9d485f-r99lx 5 | 1666592712,24727552,webhosting-operator-6b5d9d485f-r99lx 6 | 1666592727,32419840,webhosting-operator-6b5d9d485f-r99lx 7 | 1666592742,32419840,webhosting-operator-6b5d9d485f-r99lx 8 | 1666592757,40894464,webhosting-operator-6b5d9d485f-r99lx 9 | 1666592772,40894464,webhosting-operator-6b5d9d485f-r99lx 10 | 1666592787,54964224,webhosting-operator-6b5d9d485f-r99lx 11 | 1666592802,54964224,webhosting-operator-6b5d9d485f-r99lx 12 | 1666592817,66109440,webhosting-operator-6b5d9d485f-r99lx 13 | 1666592832,66109440,webhosting-operator-6b5d9d485f-r99lx 14 | 1666592847,81649664,webhosting-operator-6b5d9d485f-r99lx 15 | 1666592862,81649664,webhosting-operator-6b5d9d485f-r99lx 16 | 1666592877,95129600,webhosting-operator-6b5d9d485f-r99lx 17 | 1666592892,95129600,webhosting-operator-6b5d9d485f-r99lx 18 | 1666592907,101515264,webhosting-operator-6b5d9d485f-r99lx 19 | 1666592922,101515264,webhosting-operator-6b5d9d485f-r99lx 20 | 1666592937,117362688,webhosting-operator-6b5d9d485f-r99lx 21 | 1666592952,117362688,webhosting-operator-6b5d9d485f-r99lx 22 | 1666592967,124010496,webhosting-operator-6b5d9d485f-r99lx 23 | 1666592982,124010496,webhosting-operator-6b5d9d485f-r99lx 24 | 1666592997,143896576,webhosting-operator-6b5d9d485f-r99lx 25 | 1666593012,143896576,webhosting-operator-6b5d9d485f-r99lx 26 | 1666593027,158842880,webhosting-operator-6b5d9d485f-r99lx 27 | 1666593042,158842880,webhosting-operator-6b5d9d485f-r99lx 28 | 1666593057,169639936,webhosting-operator-6b5d9d485f-r99lx 29 | 1666593072,169639936,webhosting-operator-6b5d9d485f-r99lx 30 | 1666593087,180375552,webhosting-operator-6b5d9d485f-r99lx 31 | 1666593102,180375552,webhosting-operator-6b5d9d485f-r99lx 32 | 1666593117,189530112,webhosting-operator-6b5d9d485f-r99lx 33 | 1666593132,189530112,webhosting-operator-6b5d9d485f-r99lx 34 | 1666593147,198127616,webhosting-operator-6b5d9d485f-r99lx 35 | 1666593162,198127616,webhosting-operator-6b5d9d485f-r99lx 36 | 1666593177,210665472,webhosting-operator-6b5d9d485f-r99lx 37 | 1666593192,210665472,webhosting-operator-6b5d9d485f-r99lx 38 | 1666593207,220684288,webhosting-operator-6b5d9d485f-r99lx 39 | 1666593222,220684288,webhosting-operator-6b5d9d485f-r99lx 40 | 1666593237,228630528,webhosting-operator-6b5d9d485f-r99lx 41 | 1666593252,228630528,webhosting-operator-6b5d9d485f-r99lx 42 | 1666593267,241057792,webhosting-operator-6b5d9d485f-r99lx 43 | 1666593282,241057792,webhosting-operator-6b5d9d485f-r99lx 44 | 1666593297,247689216,webhosting-operator-6b5d9d485f-r99lx 45 | 1666593312,247689216,webhosting-operator-6b5d9d485f-r99lx 46 | 1666593327,261091328,webhosting-operator-6b5d9d485f-r99lx 47 | 1666593342,261091328,webhosting-operator-6b5d9d485f-r99lx 48 | 1666593357,269336576,webhosting-operator-6b5d9d485f-r99lx 49 | 1666593372,269336576,webhosting-operator-6b5d9d485f-r99lx 50 | 1666593387,280825856,webhosting-operator-6b5d9d485f-r99lx 51 | 1666593402,280825856,webhosting-operator-6b5d9d485f-r99lx 52 | 1666593417,280961024,webhosting-operator-6b5d9d485f-r99lx 53 | 1666593432,280961024,webhosting-operator-6b5d9d485f-r99lx 54 | 1666593447,280961024,webhosting-operator-6b5d9d485f-r99lx 55 | -------------------------------------------------------------------------------- /results/base/singleton-network_transmit.csv: -------------------------------------------------------------------------------- 1 | ts,value,pod 2 | 1666592667,1086.0333333333333,webhosting-operator-6b5d9d485f-r99lx 3 | 1666592682,1086.0333333333333,webhosting-operator-6b5d9d485f-r99lx 4 | 1666592697,1134.6333333333334,webhosting-operator-6b5d9d485f-r99lx 5 | 1666592712,1134.6333333333334,webhosting-operator-6b5d9d485f-r99lx 6 | 1666592727,278599.7,webhosting-operator-6b5d9d485f-r99lx 7 | 1666592742,278599.7,webhosting-operator-6b5d9d485f-r99lx 8 | 1666592757,583962.3333333334,webhosting-operator-6b5d9d485f-r99lx 9 | 1666592772,583962.3333333334,webhosting-operator-6b5d9d485f-r99lx 10 | 1666592787,349096.36666666664,webhosting-operator-6b5d9d485f-r99lx 11 | 1666592802,349096.36666666664,webhosting-operator-6b5d9d485f-r99lx 12 | 1666592817,668342.0333333333,webhosting-operator-6b5d9d485f-r99lx 13 | 1666592832,668342.0333333333,webhosting-operator-6b5d9d485f-r99lx 14 | 1666592847,760711.6333333333,webhosting-operator-6b5d9d485f-r99lx 15 | 1666592862,760711.6333333333,webhosting-operator-6b5d9d485f-r99lx 16 | 1666592877,649050.1333333333,webhosting-operator-6b5d9d485f-r99lx 17 | 1666592892,649050.1333333333,webhosting-operator-6b5d9d485f-r99lx 18 | 1666592907,785654.6,webhosting-operator-6b5d9d485f-r99lx 19 | 1666592922,785654.6,webhosting-operator-6b5d9d485f-r99lx 20 | 1666592937,773050.3666666667,webhosting-operator-6b5d9d485f-r99lx 21 | 1666592952,773050.3666666667,webhosting-operator-6b5d9d485f-r99lx 22 | 1666592967,764661.8333333334,webhosting-operator-6b5d9d485f-r99lx 23 | 1666592982,764661.8333333334,webhosting-operator-6b5d9d485f-r99lx 24 | 1666592997,873640.1333333333,webhosting-operator-6b5d9d485f-r99lx 25 | 1666593012,873640.1333333333,webhosting-operator-6b5d9d485f-r99lx 26 | 1666593027,752943.9,webhosting-operator-6b5d9d485f-r99lx 27 | 1666593042,752943.9,webhosting-operator-6b5d9d485f-r99lx 28 | 1666593057,706251.4,webhosting-operator-6b5d9d485f-r99lx 29 | 1666593072,706251.4,webhosting-operator-6b5d9d485f-r99lx 30 | 1666593087,865446.8333333334,webhosting-operator-6b5d9d485f-r99lx 31 | 1666593102,865446.8333333334,webhosting-operator-6b5d9d485f-r99lx 32 | 1666593117,893644.6333333333,webhosting-operator-6b5d9d485f-r99lx 33 | 1666593132,893644.6333333333,webhosting-operator-6b5d9d485f-r99lx 34 | 1666593147,403702.2,webhosting-operator-6b5d9d485f-r99lx 35 | 1666593162,403702.2,webhosting-operator-6b5d9d485f-r99lx 36 | 1666593177,1206411.5,webhosting-operator-6b5d9d485f-r99lx 37 | 1666593192,1206411.5,webhosting-operator-6b5d9d485f-r99lx 38 | 1666593207,546079.3,webhosting-operator-6b5d9d485f-r99lx 39 | 1666593222,546079.3,webhosting-operator-6b5d9d485f-r99lx 40 | 1666593237,958079.3,webhosting-operator-6b5d9d485f-r99lx 41 | 1666593252,958079.3,webhosting-operator-6b5d9d485f-r99lx 42 | 1666593267,640961.3,webhosting-operator-6b5d9d485f-r99lx 43 | 1666593282,640961.3,webhosting-operator-6b5d9d485f-r99lx 44 | 1666593297,784500.6,webhosting-operator-6b5d9d485f-r99lx 45 | 1666593312,784500.6,webhosting-operator-6b5d9d485f-r99lx 46 | 1666593327,763313.4333333333,webhosting-operator-6b5d9d485f-r99lx 47 | 1666593342,763313.4333333333,webhosting-operator-6b5d9d485f-r99lx 48 | 1666593357,721241.8,webhosting-operator-6b5d9d485f-r99lx 49 | 1666593372,721241.8,webhosting-operator-6b5d9d485f-r99lx 50 | 1666593387,875329.0666666667,webhosting-operator-6b5d9d485f-r99lx 51 | 1666593402,875329.0666666667,webhosting-operator-6b5d9d485f-r99lx 52 | 1666593417,925924.7,webhosting-operator-6b5d9d485f-r99lx 53 | 1666593432,925924.7,webhosting-operator-6b5d9d485f-r99lx 54 | 1666593447,898970.3,webhosting-operator-6b5d9d485f-r99lx 55 | -------------------------------------------------------------------------------- /results/base/singleton-network_receive.csv: -------------------------------------------------------------------------------- 1 | ts,value,pod 2 | 1666592667,1691.4,webhosting-operator-6b5d9d485f-r99lx 3 | 1666592682,1691.4,webhosting-operator-6b5d9d485f-r99lx 4 | 1666592697,4250.266666666666,webhosting-operator-6b5d9d485f-r99lx 5 | 1666592712,4250.266666666666,webhosting-operator-6b5d9d485f-r99lx 6 | 1666592727,597901.3666666667,webhosting-operator-6b5d9d485f-r99lx 7 | 1666592742,597901.3666666667,webhosting-operator-6b5d9d485f-r99lx 8 | 1666592757,1258331.9,webhosting-operator-6b5d9d485f-r99lx 9 | 1666592772,1258331.9,webhosting-operator-6b5d9d485f-r99lx 10 | 1666592787,757256.7,webhosting-operator-6b5d9d485f-r99lx 11 | 1666592802,757256.7,webhosting-operator-6b5d9d485f-r99lx 12 | 1666592817,1451474.7333333334,webhosting-operator-6b5d9d485f-r99lx 13 | 1666592832,1451474.7333333334,webhosting-operator-6b5d9d485f-r99lx 14 | 1666592847,1657433,webhosting-operator-6b5d9d485f-r99lx 15 | 1666592862,1657433,webhosting-operator-6b5d9d485f-r99lx 16 | 1666592877,1422366.3666666667,webhosting-operator-6b5d9d485f-r99lx 17 | 1666592892,1422366.3666666667,webhosting-operator-6b5d9d485f-r99lx 18 | 1666592907,1752979.6333333333,webhosting-operator-6b5d9d485f-r99lx 19 | 1666592922,1752979.6333333333,webhosting-operator-6b5d9d485f-r99lx 20 | 1666592937,1770750.5666666667,webhosting-operator-6b5d9d485f-r99lx 21 | 1666592952,1770750.5666666667,webhosting-operator-6b5d9d485f-r99lx 22 | 1666592967,1875131.7333333334,webhosting-operator-6b5d9d485f-r99lx 23 | 1666592982,1875131.7333333334,webhosting-operator-6b5d9d485f-r99lx 24 | 1666592997,2142605.1333333333,webhosting-operator-6b5d9d485f-r99lx 25 | 1666593012,2142605.1333333333,webhosting-operator-6b5d9d485f-r99lx 26 | 1666593027,1858900.4666666666,webhosting-operator-6b5d9d485f-r99lx 27 | 1666593042,1858900.4666666666,webhosting-operator-6b5d9d485f-r99lx 28 | 1666593057,1745489.9666666666,webhosting-operator-6b5d9d485f-r99lx 29 | 1666593072,1745489.9666666666,webhosting-operator-6b5d9d485f-r99lx 30 | 1666593087,2139502.033333333,webhosting-operator-6b5d9d485f-r99lx 31 | 1666593102,2139502.033333333,webhosting-operator-6b5d9d485f-r99lx 32 | 1666593117,2205777.066666667,webhosting-operator-6b5d9d485f-r99lx 33 | 1666593132,2205777.066666667,webhosting-operator-6b5d9d485f-r99lx 34 | 1666593147,995816.1333333333,webhosting-operator-6b5d9d485f-r99lx 35 | 1666593162,995816.1333333333,webhosting-operator-6b5d9d485f-r99lx 36 | 1666593177,2976528.066666667,webhosting-operator-6b5d9d485f-r99lx 37 | 1666593192,2976528.066666667,webhosting-operator-6b5d9d485f-r99lx 38 | 1666593207,1353260.8,webhosting-operator-6b5d9d485f-r99lx 39 | 1666593222,1353260.8,webhosting-operator-6b5d9d485f-r99lx 40 | 1666593237,2375572.066666667,webhosting-operator-6b5d9d485f-r99lx 41 | 1666593252,2375572.066666667,webhosting-operator-6b5d9d485f-r99lx 42 | 1666593267,1585905.6666666667,webhosting-operator-6b5d9d485f-r99lx 43 | 1666593282,1585905.6666666667,webhosting-operator-6b5d9d485f-r99lx 44 | 1666593297,1955619.6666666667,webhosting-operator-6b5d9d485f-r99lx 45 | 1666593312,1955619.6666666667,webhosting-operator-6b5d9d485f-r99lx 46 | 1666593327,1895160.2333333334,webhosting-operator-6b5d9d485f-r99lx 47 | 1666593342,1895160.2333333334,webhosting-operator-6b5d9d485f-r99lx 48 | 1666593357,1774427.0333333334,webhosting-operator-6b5d9d485f-r99lx 49 | 1666593372,1774427.0333333334,webhosting-operator-6b5d9d485f-r99lx 50 | 1666593387,2124744.1,webhosting-operator-6b5d9d485f-r99lx 51 | 1666593402,2124744.1,webhosting-operator-6b5d9d485f-r99lx 52 | 1666593417,2130822.3333333335,webhosting-operator-6b5d9d485f-r99lx 53 | 1666593432,2130822.3333333335,webhosting-operator-6b5d9d485f-r99lx 54 | 1666593447,2045811.5,webhosting-operator-6b5d9d485f-r99lx 55 | -------------------------------------------------------------------------------- /results/base/singleton-cpu.csv: -------------------------------------------------------------------------------- 1 | ts,value,pod 2 | 1666592667,0.0013973259887737498,webhosting-operator-6b5d9d485f-r99lx 3 | 1666592682,0.0016612080700237496,webhosting-operator-6b5d9d485f-r99lx 4 | 1666592697,0.002548966144444444,webhosting-operator-6b5d9d485f-r99lx 5 | 1666592712,0.002548966144444444,webhosting-operator-6b5d9d485f-r99lx 6 | 1666592727,0.06827928473219326,webhosting-operator-6b5d9d485f-r99lx 7 | 1666592742,0.07875323418080435,webhosting-operator-6b5d9d485f-r99lx 8 | 1666592757,0.1445208806123521,webhosting-operator-6b5d9d485f-r99lx 9 | 1666592772,0.16701345784429658,webhosting-operator-6b5d9d485f-r99lx 10 | 1666592787,0.22448712641054433,webhosting-operator-6b5d9d485f-r99lx 11 | 1666592802,0.2595462959258221,webhosting-operator-6b5d9d485f-r99lx 12 | 1666592817,0.3453067695649856,webhosting-operator-6b5d9d485f-r99lx 13 | 1666592832,0.3559755070666667,webhosting-operator-6b5d9d485f-r99lx 14 | 1666592847,0.41205392192222223,webhosting-operator-6b5d9d485f-r99lx 15 | 1666592862,0.41205392192222223,webhosting-operator-6b5d9d485f-r99lx 16 | 1666592877,0.4559710225777778,webhosting-operator-6b5d9d485f-r99lx 17 | 1666592892,0.4559710225777778,webhosting-operator-6b5d9d485f-r99lx 18 | 1666592907,0.42048143639999996,webhosting-operator-6b5d9d485f-r99lx 19 | 1666592922,0.42048143639999996,webhosting-operator-6b5d9d485f-r99lx 20 | 1666592937,0.43078289096666655,webhosting-operator-6b5d9d485f-r99lx 21 | 1666592952,0.43078289096666655,webhosting-operator-6b5d9d485f-r99lx 22 | 1666592967,0.39762051187777775,webhosting-operator-6b5d9d485f-r99lx 23 | 1666592982,0.39762051187777775,webhosting-operator-6b5d9d485f-r99lx 24 | 1666592997,0.5142313253777778,webhosting-operator-6b5d9d485f-r99lx 25 | 1666593012,0.5142313253777778,webhosting-operator-6b5d9d485f-r99lx 26 | 1666593027,0.5037625069555555,webhosting-operator-6b5d9d485f-r99lx 27 | 1666593042,0.5037625069555555,webhosting-operator-6b5d9d485f-r99lx 28 | 1666593057,0.5633891238666665,webhosting-operator-6b5d9d485f-r99lx 29 | 1666593072,0.5633891238666665,webhosting-operator-6b5d9d485f-r99lx 30 | 1666593087,0.47923826943333314,webhosting-operator-6b5d9d485f-r99lx 31 | 1666593102,0.47923826943333314,webhosting-operator-6b5d9d485f-r99lx 32 | 1666593117,0.47241126398888894,webhosting-operator-6b5d9d485f-r99lx 33 | 1666593132,0.47241126398888894,webhosting-operator-6b5d9d485f-r99lx 34 | 1666593147,0.4296189880222225,webhosting-operator-6b5d9d485f-r99lx 35 | 1666593162,0.4296189880222225,webhosting-operator-6b5d9d485f-r99lx 36 | 1666593177,0.46061833982222233,webhosting-operator-6b5d9d485f-r99lx 37 | 1666593192,0.46061833982222233,webhosting-operator-6b5d9d485f-r99lx 38 | 1666593207,0.5269689206222223,webhosting-operator-6b5d9d485f-r99lx 39 | 1666593222,0.5269689206222223,webhosting-operator-6b5d9d485f-r99lx 40 | 1666593237,0.5091103374999998,webhosting-operator-6b5d9d485f-r99lx 41 | 1666593252,0.5091103374999998,webhosting-operator-6b5d9d485f-r99lx 42 | 1666593267,0.5195348532666666,webhosting-operator-6b5d9d485f-r99lx 43 | 1666593282,0.5195348532666666,webhosting-operator-6b5d9d485f-r99lx 44 | 1666593297,0.4573483411444442,webhosting-operator-6b5d9d485f-r99lx 45 | 1666593312,0.4573483411444442,webhosting-operator-6b5d9d485f-r99lx 46 | 1666593327,0.5046873200555558,webhosting-operator-6b5d9d485f-r99lx 47 | 1666593342,0.5046873200555558,webhosting-operator-6b5d9d485f-r99lx 48 | 1666593357,0.47191863308888865,webhosting-operator-6b5d9d485f-r99lx 49 | 1666593372,0.47191863308888865,webhosting-operator-6b5d9d485f-r99lx 50 | 1666593387,0.5060672277888891,webhosting-operator-6b5d9d485f-r99lx 51 | 1666593402,0.5060672277888891,webhosting-operator-6b5d9d485f-r99lx 52 | 1666593417,0.475237114744444,webhosting-operator-6b5d9d485f-r99lx 53 | 1666593432,0.475237114744444,webhosting-operator-6b5d9d485f-r99lx 54 | 1666593447,0.558265907466667,webhosting-operator-6b5d9d485f-r99lx 55 | -------------------------------------------------------------------------------- /pandoc/highlight-style.theme: -------------------------------------------------------------------------------- 1 | { 2 | "text-color": null, 3 | "background-color": "#f8f8f8", 4 | "line-number-color": "#aaaaaa", 5 | "line-number-background-color": null, 6 | "text-styles": { 7 | "Alert": { 8 | "text-color": "#ff0000", 9 | "background-color": null, 10 | "bold": true, 11 | "italic": false, 12 | "underline": false 13 | }, 14 | "Annotation": { 15 | "text-color": "#60a0b0", 16 | "background-color": null, 17 | "bold": true, 18 | "italic": true, 19 | "underline": false 20 | }, 21 | "Attribute": { 22 | "text-color": null, 23 | "background-color": null, 24 | "bold": false, 25 | "italic": false, 26 | "underline": false 27 | }, 28 | "BaseN": { 29 | "text-color": "#40a070", 30 | "background-color": null, 31 | "bold": false, 32 | "italic": false, 33 | "underline": false 34 | }, 35 | "BuiltIn": { 36 | "text-color": null, 37 | "background-color": null, 38 | "bold": false, 39 | "italic": false, 40 | "underline": false 41 | }, 42 | "Char": { 43 | "text-color": "#4070a0", 44 | "background-color": null, 45 | "bold": false, 46 | "italic": false, 47 | "underline": false 48 | }, 49 | "Comment": { 50 | "text-color": "#898887", 51 | "background-color": null, 52 | "bold": false, 53 | "italic": false, 54 | "underline": false 55 | }, 56 | "CommentVar": { 57 | "text-color": "#60a0b0", 58 | "background-color": null, 59 | "bold": true, 60 | "italic": true, 61 | "underline": false 62 | }, 63 | "Constant": { 64 | "text-color": "#880000", 65 | "background-color": null, 66 | "bold": false, 67 | "italic": false, 68 | "underline": false 69 | }, 70 | "ControlFlow": { 71 | "text-color": "#007020", 72 | "background-color": null, 73 | "bold": true, 74 | "italic": false, 75 | "underline": false 76 | }, 77 | "DataType": { 78 | "text-color": "#902000", 79 | "background-color": null, 80 | "bold": false, 81 | "italic": false, 82 | "underline": false 83 | }, 84 | "DecVal": { 85 | "text-color": "#40a070", 86 | "background-color": null, 87 | "bold": false, 88 | "italic": false, 89 | "underline": false 90 | }, 91 | "Documentation": { 92 | "text-color": "#ba2121", 93 | "background-color": null, 94 | "bold": false, 95 | "italic": true, 96 | "underline": false 97 | }, 98 | "Error": { 99 | "text-color": "#ff0000", 100 | "background-color": null, 101 | "bold": true, 102 | "italic": false, 103 | "underline": false 104 | }, 105 | "Extension": { 106 | "text-color": null, 107 | "background-color": null, 108 | "bold": false, 109 | "italic": false, 110 | "underline": false 111 | }, 112 | "Float": { 113 | "text-color": "#40a070", 114 | "background-color": null, 115 | "bold": false, 116 | "italic": false, 117 | "underline": false 118 | }, 119 | "Function": { 120 | "text-color": "#0032b3", 121 | "background-color": null, 122 | "bold": false, 123 | "italic": false, 124 | "underline": false 125 | }, 126 | "Import": { 127 | "text-color": null, 128 | "background-color": null, 129 | "bold": false, 130 | "italic": false, 131 | "underline": false 132 | }, 133 | "Information": { 134 | "text-color": "#60a0b0", 135 | "background-color": null, 136 | "bold": true, 137 | "italic": true, 138 | "underline": false 139 | }, 140 | "Keyword": { 141 | "text-color": "#007020", 142 | "background-color": null, 143 | "bold": true, 144 | "italic": false, 145 | "underline": false 146 | }, 147 | "Operator": { 148 | "text-color": "#666666", 149 | "background-color": null, 150 | "bold": false, 151 | "italic": false, 152 | "underline": false 153 | }, 154 | "Other": { 155 | "text-color": "#0032b3", 156 | "background-color": null, 157 | "bold": false, 158 | "italic": false, 159 | "underline": false 160 | }, 161 | "Preprocessor": { 162 | "text-color": null, 163 | "background-color": null, 164 | "bold": false, 165 | "italic": false, 166 | "underline": false 167 | }, 168 | "SpecialChar": { 169 | "text-color": "#4070a0", 170 | "background-color": null, 171 | "bold": false, 172 | "italic": false, 173 | "underline": false 174 | }, 175 | "SpecialString": { 176 | "text-color": "#bb6688", 177 | "background-color": null, 178 | "bold": false, 179 | "italic": false, 180 | "underline": false 181 | }, 182 | "String": { 183 | "text-color": "#067d16", 184 | "background-color": null, 185 | "bold": false, 186 | "italic": false, 187 | "underline": false 188 | }, 189 | "Variable": { 190 | "text-color": "#19177c", 191 | "background-color": null, 192 | "bold": false, 193 | "italic": false, 194 | "underline": false 195 | }, 196 | "VerbatimString": { 197 | "text-color": "#4070a0", 198 | "background-color": null, 199 | "bold": false, 200 | "italic": false, 201 | "underline": false 202 | }, 203 | "Warning": { 204 | "text-color": "#60a0b0", 205 | "background-color": null, 206 | "bold": true, 207 | "italic": true, 208 | "underline": false 209 | } 210 | } 211 | } 212 | -------------------------------------------------------------------------------- /content/70-conclusion.md: -------------------------------------------------------------------------------- 1 | # Conclusion and Future Work 2 | 3 | This thesis performed the first steps toward horizontally scalable Kubernetes controllers. 4 | As part of the motivation ([chapter @sec:motivation]), it was shown that the current leader election mechanisms effectively prevent scaling Kubernetes controllers horizontally. 5 | Because of this, Kubernetes controllers can only be scaled vertically which restricts their scalability to the available machine sizes and network bandwidth of the underlying infrastructure. 6 | After laying out the relevant background of Kubernetes controllers for this thesis, important proven sharding mechanisms of distributed databases were summarized ([chapter @sec:background]). 7 | This served as a basis for analyzing what is required for removing the current scalability limitations of Kubernetes controllers and scaling them horizontally ([chapter @sec:requirement-analysis]). 8 | Based on this analysis, a design was presented that applies well-known sharding mechanisms from the field of distributed databases to the specific problem of sharding in Kubernetes controllers. 9 | The proposed design gives an architectural overview and then describes the details of how the listed requirements can be fulfilled ([chapter @sec:design]). 10 | The second half of the thesis described how the proposed design is implemented and evaluated. 11 | For this, the webhosting operator was implemented as an example operator for demonstrating the sharding mechanisms in practice. 12 | The operator was implemented using the controller-runtime library, which was enhanced to generically realize the presented sharding design ([chapter @sec:implementation]). 13 | In the last chapter, the implementation was used as a basis for evaluation in an experiment. 14 | After describing the deployment and monitoring setup in detail, a load test experiment was conducted to measure how well the sharded operator scales horizontally ([chapter @sec:evaluation]). 15 | 16 | To summarize, this thesis showed that well-known sharding mechanisms from the field of distributed databases can be applied well to Kubernetes controllers. 17 | The resulting design allows for overcoming the current scalability limitation of controllers and scaling them horizontally. 18 | The conducted evaluation shows that the presented sharding implementation works well and realizes a good distribution of responsibility for objects among multiple controller instances. 19 | Furthermore, the resource requirements of the singleton controller are distributed across multiple shards. 20 | However, the sharder instance still has a high resource usage and thus faces similar scalability limitations in the vertical direction as the singleton controller setup ([@sec:results]). 21 | Nevertheless, the measurements lead to the conclusion that this limitation can be removed by deploying the sharder components separately from the shard components ([@sec:discussion]). 22 | 23 | Apart from this improvement, this thesis lays the ground for future work in several areas: further optimizations, solving different challenges, evaluation of more aspects, and advanced applications. 24 | First, it should be investigated how the use of webhooks for sharded API objects could improve the design and reduce the resource requirements of the sharder components. 25 | E.g., a mutating webhook could be used to assign API objects to shards during admission instead of in a dedicated controller. 26 | Furthermore, the proposed design faces the challenge that the `shard` and `drain` labels on API objects should not be mutated by humans. 27 | If operators grant end users access to the Kubernetes API, validating webhooks could be leveraged to deny user requests that mutate or remove sharding-related labels. 28 | Another challenge is, that a single component might run multiple controllers that act on the same object kind but can't share the same sharded cache. 29 | In this case, it needs to be investigated how this challenge could be solved by introducing dedicated caches with different selectors. 30 | Additionally, some controllers evaluate relationships between objects of the same kind. 31 | E.g., the Kubernetes scheduler considers scheduling constraints in form of inter-pod anti-affinities that allow the spreading of related pods across different failure domains. 32 | Future work could investigate whether sharding mechanisms can be applied to such use cases as well. 33 | 34 | Furthermore, it should be evaluated what impact the presented sharding mechanisms have on the resource requirements and scalability of Kubernetes control plane components, i.e. the Kubernetes API server and etcd ([@tbl:scaling-resources-server]). 35 | Also, this thesis only evaluated the resource requirements of a static set of sharded controller instances. 36 | Future work could conduct experiments that additionally perform rolling updates of the operator `Deployment` or scale-out/in the operator. 37 | The impact of reassignment operations during such changes to the instance set needs to be investigated and possibly further optimizations of the design need to be implemented. 38 | 39 | Last, future work could evaluate advanced applications of the presented sharding mechanisms. 40 | E.g., the sharding design could be implemented in Kubernetes core components like the controller manager. 41 | The Kubernetes core controllers are not based on the controller-runtime library [@k8s], hence they can't reuse the implementation done for this thesis. 42 | Nevertheless, the proposed design could still be applied to these controllers as well. 43 | Another idea for future work is to investigate how sharded controllers could be scaled up or down automatically based on utilization. 44 | E.g., a `HorizontalPodAutoscaler` [@k8sdocs] could be introduced which adds more controller instances when the average queue wait duration exceeds a certain target or removes instances when most of the controller workers are inactive. 45 | Furthermore, another advanced application of the presented sharding mechanisms is to make the number of virtual nodes per controller instance configurable and thereby control the expected distribution of objects. 46 | This could be used to implement canary rollout approaches for controllers [@schermann2018; @adams2015practice]. 47 | I.e., when a new controller version is rolled out, only a small subset of objects is assigned to the new version. 48 | The object distribution could then be gradually shifted towards the new version to increase the operator's confidence in the new version. 49 | 50 | To conclude, this thesis laid the ground and performed the first steps toward horizontally scalable Kubernetes controllers. 51 | It shows that this can be achieved by applying proven sharding mechanisms used in distributed databases to Kubernetes controllers. 52 | Performing sharding for Kubernetes controllers comes with a certain overhead, as any other sharding mechanism does as well. 53 | Hence, it is not relevant for small controllers or clusters. 54 | However, it proves to be interesting for scaling controllers and clusters further that are already heavily loaded and running into scalability limitations. 55 | The presented design and implementation are not ready for production deployments yet, but important next steps have been listed that can be performed to make Kubernetes controller sharding production-ready. 56 | -------------------------------------------------------------------------------- /results/base/sharded-websites_per_shard.csv: -------------------------------------------------------------------------------- 1 | ts,value,shard 2 | 1666596649,0,webhosting-operator-559494f768-7cjp8 3 | 1666596664,0,webhosting-operator-559494f768-7cjp8 4 | 1666596679,0,webhosting-operator-559494f768-7cjp8 5 | 1666596694,39,webhosting-operator-559494f768-7cjp8 6 | 1666596709,129,webhosting-operator-559494f768-7cjp8 7 | 1666596724,165,webhosting-operator-559494f768-7cjp8 8 | 1666596739,276,webhosting-operator-559494f768-7cjp8 9 | 1666596754,330,webhosting-operator-559494f768-7cjp8 10 | 1666596769,408,webhosting-operator-559494f768-7cjp8 11 | 1666596784,464,webhosting-operator-559494f768-7cjp8 12 | 1666596799,553,webhosting-operator-559494f768-7cjp8 13 | 1666596814,594,webhosting-operator-559494f768-7cjp8 14 | 1666596829,675,webhosting-operator-559494f768-7cjp8 15 | 1666596844,719,webhosting-operator-559494f768-7cjp8 16 | 1666596859,815,webhosting-operator-559494f768-7cjp8 17 | 1666596874,857,webhosting-operator-559494f768-7cjp8 18 | 1666596889,959,webhosting-operator-559494f768-7cjp8 19 | 1666596904,1002,webhosting-operator-559494f768-7cjp8 20 | 1666596919,1090,webhosting-operator-559494f768-7cjp8 21 | 1666596934,1129,webhosting-operator-559494f768-7cjp8 22 | 1666596949,1228,webhosting-operator-559494f768-7cjp8 23 | 1666596964,1273,webhosting-operator-559494f768-7cjp8 24 | 1666596979,1366,webhosting-operator-559494f768-7cjp8 25 | 1666596994,1422,webhosting-operator-559494f768-7cjp8 26 | 1666597009,1506,webhosting-operator-559494f768-7cjp8 27 | 1666597024,1552,webhosting-operator-559494f768-7cjp8 28 | 1666597039,1658,webhosting-operator-559494f768-7cjp8 29 | 1666597054,1708,webhosting-operator-559494f768-7cjp8 30 | 1666597069,1803,webhosting-operator-559494f768-7cjp8 31 | 1666597084,1843,webhosting-operator-559494f768-7cjp8 32 | 1666597099,1937,webhosting-operator-559494f768-7cjp8 33 | 1666597114,1982,webhosting-operator-559494f768-7cjp8 34 | 1666597129,2058,webhosting-operator-559494f768-7cjp8 35 | 1666597144,2101,webhosting-operator-559494f768-7cjp8 36 | 1666597159,2193,webhosting-operator-559494f768-7cjp8 37 | 1666597174,2237,webhosting-operator-559494f768-7cjp8 38 | 1666597189,2340,webhosting-operator-559494f768-7cjp8 39 | 1666597204,2388,webhosting-operator-559494f768-7cjp8 40 | 1666597219,2475,webhosting-operator-559494f768-7cjp8 41 | 1666597234,2508,webhosting-operator-559494f768-7cjp8 42 | 1666597249,2599,webhosting-operator-559494f768-7cjp8 43 | 1666597264,2637,webhosting-operator-559494f768-7cjp8 44 | 1666597279,2732,webhosting-operator-559494f768-7cjp8 45 | 1666597294,2743,webhosting-operator-559494f768-7cjp8 46 | 1666597309,2743,webhosting-operator-559494f768-7cjp8 47 | 1666597324,2743,webhosting-operator-559494f768-7cjp8 48 | 1666597339,2743,webhosting-operator-559494f768-7cjp8 49 | 1666597354,2743,webhosting-operator-559494f768-7cjp8 50 | 1666597369,2743,webhosting-operator-559494f768-7cjp8 51 | 1666597384,2743,webhosting-operator-559494f768-7cjp8 52 | 1666597399,2743,webhosting-operator-559494f768-7cjp8 53 | 1666597414,2743,webhosting-operator-559494f768-7cjp8 54 | 1666597429,2743,webhosting-operator-559494f768-7cjp8 55 | 1666596649,0,webhosting-operator-559494f768-rbsdv 56 | 1666596664,0,webhosting-operator-559494f768-rbsdv 57 | 1666596679,0,webhosting-operator-559494f768-rbsdv 58 | 1666596694,43,webhosting-operator-559494f768-rbsdv 59 | 1666596709,133,webhosting-operator-559494f768-rbsdv 60 | 1666596724,180,webhosting-operator-559494f768-rbsdv 61 | 1666596739,254,webhosting-operator-559494f768-rbsdv 62 | 1666596754,293,webhosting-operator-559494f768-rbsdv 63 | 1666596769,393,webhosting-operator-559494f768-rbsdv 64 | 1666596784,425,webhosting-operator-559494f768-rbsdv 65 | 1666596799,520,webhosting-operator-559494f768-rbsdv 66 | 1666596814,568,webhosting-operator-559494f768-rbsdv 67 | 1666596829,670,webhosting-operator-559494f768-rbsdv 68 | 1666596844,714,webhosting-operator-559494f768-rbsdv 69 | 1666596859,802,webhosting-operator-559494f768-rbsdv 70 | 1666596874,848,webhosting-operator-559494f768-rbsdv 71 | 1666596889,931,webhosting-operator-559494f768-rbsdv 72 | 1666596904,986,webhosting-operator-559494f768-rbsdv 73 | 1666596919,1089,webhosting-operator-559494f768-rbsdv 74 | 1666596934,1131,webhosting-operator-559494f768-rbsdv 75 | 1666596949,1216,webhosting-operator-559494f768-rbsdv 76 | 1666596964,1254,webhosting-operator-559494f768-rbsdv 77 | 1666596979,1340,webhosting-operator-559494f768-rbsdv 78 | 1666596994,1377,webhosting-operator-559494f768-rbsdv 79 | 1666597009,1460,webhosting-operator-559494f768-rbsdv 80 | 1666597024,1500,webhosting-operator-559494f768-rbsdv 81 | 1666597039,1586,webhosting-operator-559494f768-rbsdv 82 | 1666597054,1636,webhosting-operator-559494f768-rbsdv 83 | 1666597069,1728,webhosting-operator-559494f768-rbsdv 84 | 1666597084,1768,webhosting-operator-559494f768-rbsdv 85 | 1666597099,1852,webhosting-operator-559494f768-rbsdv 86 | 1666597114,1898,webhosting-operator-559494f768-rbsdv 87 | 1666597129,1994,webhosting-operator-559494f768-rbsdv 88 | 1666597144,2026,webhosting-operator-559494f768-rbsdv 89 | 1666597159,2122,webhosting-operator-559494f768-rbsdv 90 | 1666597174,2152,webhosting-operator-559494f768-rbsdv 91 | 1666597189,2245,webhosting-operator-559494f768-rbsdv 92 | 1666597204,2295,webhosting-operator-559494f768-rbsdv 93 | 1666597219,2378,webhosting-operator-559494f768-rbsdv 94 | 1666597234,2425,webhosting-operator-559494f768-rbsdv 95 | 1666597249,2511,webhosting-operator-559494f768-rbsdv 96 | 1666597264,2555,webhosting-operator-559494f768-rbsdv 97 | 1666597279,2634,webhosting-operator-559494f768-rbsdv 98 | 1666597294,2638,webhosting-operator-559494f768-rbsdv 99 | 1666597309,2638,webhosting-operator-559494f768-rbsdv 100 | 1666597324,2638,webhosting-operator-559494f768-rbsdv 101 | 1666597339,2638,webhosting-operator-559494f768-rbsdv 102 | 1666597354,2638,webhosting-operator-559494f768-rbsdv 103 | 1666597369,2638,webhosting-operator-559494f768-rbsdv 104 | 1666597384,2638,webhosting-operator-559494f768-rbsdv 105 | 1666597399,2638,webhosting-operator-559494f768-rbsdv 106 | 1666597414,2638,webhosting-operator-559494f768-rbsdv 107 | 1666597429,2638,webhosting-operator-559494f768-rbsdv 108 | 1666596649,0,webhosting-operator-559494f768-sk4mv 109 | 1666596664,0,webhosting-operator-559494f768-sk4mv 110 | 1666596679,0,webhosting-operator-559494f768-sk4mv 111 | 1666596694,39,webhosting-operator-559494f768-sk4mv 112 | 1666596709,119,webhosting-operator-559494f768-sk4mv 113 | 1666596724,166,webhosting-operator-559494f768-sk4mv 114 | 1666596739,241,webhosting-operator-559494f768-sk4mv 115 | 1666596754,278,webhosting-operator-559494f768-sk4mv 116 | 1666596769,360,webhosting-operator-559494f768-sk4mv 117 | 1666596784,402,webhosting-operator-559494f768-sk4mv 118 | 1666596799,478,webhosting-operator-559494f768-sk4mv 119 | 1666596814,519,webhosting-operator-559494f768-sk4mv 120 | 1666596829,596,webhosting-operator-559494f768-sk4mv 121 | 1666596844,638,webhosting-operator-559494f768-sk4mv 122 | 1666596859,714,webhosting-operator-559494f768-sk4mv 123 | 1666596874,756,webhosting-operator-559494f768-sk4mv 124 | 1666596889,831,webhosting-operator-559494f768-sk4mv 125 | 1666596904,863,webhosting-operator-559494f768-sk4mv 126 | 1666596919,932,webhosting-operator-559494f768-sk4mv 127 | 1666596934,981,webhosting-operator-559494f768-sk4mv 128 | 1666596949,1057,webhosting-operator-559494f768-sk4mv 129 | 1666596964,1103,webhosting-operator-559494f768-sk4mv 130 | 1666596979,1185,webhosting-operator-559494f768-sk4mv 131 | 1666596994,1221,webhosting-operator-559494f768-sk4mv 132 | 1666597009,1315,webhosting-operator-559494f768-sk4mv 133 | 1666597024,1359,webhosting-operator-559494f768-sk4mv 134 | 1666597039,1427,webhosting-operator-559494f768-sk4mv 135 | 1666597054,1457,webhosting-operator-559494f768-sk4mv 136 | 1666597069,1515,webhosting-operator-559494f768-sk4mv 137 | 1666597084,1565,webhosting-operator-559494f768-sk4mv 138 | 1666597099,1648,webhosting-operator-559494f768-sk4mv 139 | 1666597114,1687,webhosting-operator-559494f768-sk4mv 140 | 1666597129,1774,webhosting-operator-559494f768-sk4mv 141 | 1666597144,1829,webhosting-operator-559494f768-sk4mv 142 | 1666597159,1901,webhosting-operator-559494f768-sk4mv 143 | 1666597174,1956,webhosting-operator-559494f768-sk4mv 144 | 1666597189,2021,webhosting-operator-559494f768-sk4mv 145 | 1666597204,2053,webhosting-operator-559494f768-sk4mv 146 | 1666597219,2143,webhosting-operator-559494f768-sk4mv 147 | 1666597234,2193,webhosting-operator-559494f768-sk4mv 148 | 1666597249,2276,webhosting-operator-559494f768-sk4mv 149 | 1666597264,2324,webhosting-operator-559494f768-sk4mv 150 | 1666597279,2410,webhosting-operator-559494f768-sk4mv 151 | 1666597294,2415,webhosting-operator-559494f768-sk4mv 152 | 1666597309,2415,webhosting-operator-559494f768-sk4mv 153 | 1666597324,2415,webhosting-operator-559494f768-sk4mv 154 | 1666597339,2415,webhosting-operator-559494f768-sk4mv 155 | 1666597354,2415,webhosting-operator-559494f768-sk4mv 156 | 1666597369,2415,webhosting-operator-559494f768-sk4mv 157 | 1666597384,2415,webhosting-operator-559494f768-sk4mv 158 | 1666597399,2415,webhosting-operator-559494f768-sk4mv 159 | 1666597414,2415,webhosting-operator-559494f768-sk4mv 160 | 1666597429,2415,webhosting-operator-559494f768-sk4mv 161 | -------------------------------------------------------------------------------- /pandoc/includes/header.tex: -------------------------------------------------------------------------------- 1 | %%% begin header.tex 2 | % \makeatletter 3 | % \providecommand*{\input@path}{} 4 | % % For latex-workshop only 5 | % \edef\input@path{{../convert/pandoc/includes/}{../pandoc/}{./../}\input@path}% 6 | % % Normal paths 7 | % \edef\input@path{{convert/pandoc/includes/}{pandoc/}{./}\input@path}% prepend 8 | % \makeatother 9 | 10 | % Language ========================================== 11 | \usepackage{microtype} 12 | %\usepackage[american]{babel} 13 | \selectlanguage{american} 14 | \input{hyphenex/ushyphex.tex} 15 | \usepackage{fontspec} 16 | 17 | % Calculations ======================================= 18 | \usepackage{calc} 19 | 20 | % Some booleans ====================================== 21 | \usepackage{ifthen} 22 | \newboolean{printVersion} 23 | \setboolean{printVersion}{true} 24 | 25 | 26 | %% Numerals ========================================== 27 | \usepackage{nth} 28 | 29 | %% Geometry ========================================== 30 | \usepackage{printlen} 31 | \newlength{\innermargin} 32 | \setlength{\innermargin}{25mm} 33 | \usepackage[nomarginpar, 34 | textwidth=155mm, 35 | textheight=240mm, 36 | bottom=30mm, 37 | headheight=15pt, 38 | headsep=20pt, 39 | footskip=30pt, 40 | bindingoffset=5mm, 41 | inner=\innermargin 42 | ]{geometry} 43 | \setlength{\columnsep}{0.95cm} 44 | \usepackage[final]{pdfpages} 45 | 46 | %% Grafics ========================================== 47 | \usepackage{graphicx} 48 | \usepackage[export]{adjustbox} 49 | \usepackage{tikz} 50 | \usepackage{wrapfig} 51 | \usepackage[absolute]{textpos} 52 | 53 | %% Caption =========================================== 54 | \usepackage[% 55 | format=plain, 56 | justification=centering, 57 | margin=12pt, 58 | font=normal, 59 | labelfont=bf 60 | ]{caption} 61 | \usepackage[]{subcaption} 62 | 63 | \DeclareCaptionStyle{caption}{ 64 | format=plain, 65 | justification=centering, 66 | margin=12pt, 67 | font=normal, 68 | labelfont=bf, 69 | position=bottom, 70 | aboveskip=10pt, 71 | belowskip=-5pt 72 | } 73 | 74 | \captionsetup[figure]{style=caption} 75 | \captionsetup[table]{style=caption} 76 | \captionsetup[listing]{style=caption} 77 | 78 | % continuous figure and table numbering 79 | \counterwithout{figure}{chapter} 80 | \counterwithout{table}{chapter} 81 | 82 | % set caption label style to arabic 83 | \renewcommand{\thefigure}{\arabic{figure}} 84 | 85 | % Footnotes 86 | \usepackage[perpage]{footmisc} 87 | % ==================================================== 88 | 89 | %% Tables ============================================ 90 | \usepackage{array} 91 | %\usepackage{ctable} 92 | \usepackage{multirow} 93 | \usepackage{tabularx} 94 | \usepackage{booktabs} 95 | \usepackage{longtable} 96 | \usepackage{supertabular} 97 | \usepackage{ltxtable} 98 | \usepackage{collcell} 99 | \usepackage{makecell} 100 | \usepackage{colortbl} 101 | \usepackage[svgnames]{xcolor} 102 | 103 | % cell padding 104 | \renewcommand{\arraystretch}{1.5} 105 | 106 | % ==================================================== 107 | 108 | %% Use Links & Refs ================================== 109 | \ifthenelse{\boolean{printVersion}}{ 110 | \colorlet{myurlcolor}{gray!50!black}% 111 | \colorlet{mylinkcolor}{gray!50!black}% 112 | \colorlet{mycitecolor}{gray!50!black}% 113 | \colorlet{myanchorcolor}{gray!50!black}% 114 | \colorlet{mybackrefcolor}{gray!10!black}% 115 | }{% 116 | \colorlet{myurlcolor}{blue!50!black}% 117 | \colorlet{mylinkcolor}{blue!50!black}% 118 | \colorlet{mycitecolor}{blue!50!black}% 119 | \colorlet{myanchorcolor}{blue!50!black}% 120 | \colorlet{mybackrefcolor}{green!35!black}% 121 | } 122 | 123 | % Loaded from pandoc 124 | % \usepackage[hidelinks, 125 | % colorlinks = true, 126 | % linkcolor = black, 127 | % urlcolor = myurlcolor, 128 | % citecolor = black, 129 | % anchorcolor = black]{hyperref} 130 | 131 | %% Code Blocks ======================================== 132 | \usepackage[outputdir=output-tex]{minted} 133 | \usemintedstyle{friendly} 134 | \renewcommand{\listingscaption}{Code} 135 | \renewcommand{\listoflistingscaption}{List of Code} 136 | 137 | \setminted{ 138 | bgcolor=WhiteSmoke, 139 | fontsize=\small, 140 | breaklines=true, 141 | tabsize=2, 142 | } 143 | \setmintedinline{ 144 | fontsize=\normalsize, 145 | } 146 | 147 | \setmonofont{JetBrainsMono}[ 148 | Path=./pandoc/fonts/, 149 | Scale=0.85, 150 | Extension = .ttf, 151 | UprightFont=*-Regular, 152 | BoldFont=*-Bold, 153 | ItalicFont=*-Italic, 154 | BoldItalicFont=*-BoldItalic 155 | ] 156 | 157 | %% Quotes ============================================= 158 | \usepackage[font=itshape,vskip=5pt]{quoting} 159 | 160 | % Hypenate tt text 161 | \usepackage[htt]{hyphenat} 162 | 163 | %% Title & Headings ================================== 164 | 165 | \renewcommand*\chapterheadstartvskip{\vspace*{0pt}} 166 | \renewcommand*\chapterheadendvskip{\vspace*{12pt}} 167 | 168 | % Tocless Command ==================================== 169 | \newcommand{\nocontentsline}[3]{} 170 | \newcommand{\tocless}[2]{\bgroup\let\addcontentsline=\nocontentsline#1{#2}\egroup} 171 | 172 | % Make some floats adjustments ======================= 173 | % make floatbarrier at each section 174 | \usepackage[section]{placeins} 175 | 176 | % spacing between floats and text/floats 177 | % \setlength{\textfloatsep}{20pt plus 2pt minus 4pt} 178 | % \setlength{\floatsep}{12pt plus 2pt minus 2pt} 179 | \setlength{\intextsep}{20pt plus 3pt minus 3pt} 180 | 181 | % \setcounter{topnumber}{2} 182 | % \setcounter{bottomnumber}{2} 183 | % \setcounter{totalnumber}{3} % 2 may work better 184 | % \setcounter{dbltopnumber}{2} % for 2-column pages 185 | \renewcommand{\topfraction}{0.85} 186 | \renewcommand{\bottomfraction}{0.8} 187 | \renewcommand{\textfraction}{0.07} 188 | \renewcommand{\floatpagefraction}{0.8} 189 | %\renewcommand{\dbltopfraction}{.66} 190 | %\renewcommand{\dblfloatpagefraction}{.8} 191 | 192 | % Allow display break 193 | %\allowdisplaybreaks[3] 194 | %===================================================== 195 | 196 | % Clever Refs 197 | \usepackage{hyperref} 198 | \usepackage{cleveref} % load it after thmtools (there is some problem) 199 | \crefname{appsec}{appendix}{appendices} 200 | 201 | \KOMAoptions{cleardoublepage=empty} 202 | 203 | %===================================================== 204 | %custom stuff ======================================== 205 | %===================================================== 206 | 207 | \usepackage{lipsum} 208 | % include list of figures, etc. in TOC 209 | \usepackage[ 210 | nottoc, % exclude TOC ref from TOC 211 | ]{tocbibind} 212 | 213 | \usepackage{todonotes} 214 | 215 | %%% custom title page 216 | \newcommand{\TitlePageTitle}{Towards Horizontally Scalable Kubernetes Controllers} 217 | \newcommand{\ShortTitle}{Towards Horizontally Scalable Kubernetes Controllers} 218 | \newcommand{\Degree}{Master of Science (M.Sc.)} 219 | \newcommand{\Course}{Informatik} 220 | \newcommand{\EnrollmentNumber}{8559152} 221 | \newcommand{\Company}{SAP SE / Schwarz IT KG} 222 | \newcommand{\Lecturer}{Prof. Dr.-Ing. habil. Dennis Pfisterer} 223 | \newcommand{\AdditionalTitlePageLogo}{} 224 | % \newcommand{\AdditionalTitlePageLogo}{\fbox{\includegraphics[width=3cm]{logo/...}}} 225 | \newcommand{\TypeOfDocument}{Studienarbeit} 226 | \newcommand{\Date}{27.06. - 26.12.2022} 227 | 228 | \makeatletter 229 | \renewcommand*{\maketitle}{% 230 | \begin{titlepage} 231 | \begin{center} 232 | \vspace*{-2cm} 233 | \AdditionalTitlePageLogo\hfill\includegraphics[width=5cm]{assets/dhbw-cas.pdf}\\[2cm] 234 | {\Huge{\TitlePageTitle}\par} 235 | \vspace{1.5cm} 236 | {\Huge\scshape \TypeOfDocument}\\[1cm] 237 | {\large für die Prüfung zum}\\[0.2cm] 238 | {\Large \Degree}\\[1.5cm] 239 | {\large im Studiengang}\\[0.2cm] 240 | {\large \Course}\\[1.5cm] 241 | {\large am Center for Advanced Studies}\\[0.2cm] 242 | {\large der Dualen Hochschule Baden-Württemberg}\\[0.5cm] 243 | {\large von}\\[0.5cm] 244 | {\large\bfseries \@author}\\[1cm] 245 | \vfill 246 | \end{center} 247 | \begin{tabular}{l@{\hspace{2cm}}l} 248 | Matrikelnummer & \EnrollmentNumber \\ 249 | Bearbeitungszeitraum & \Date \\ 250 | Dualer Partner & \Company \\ 251 | Prüfer & \Lecturer \\ 252 | \end{tabular} 253 | \end{titlepage} 254 | } 255 | \makeatother 256 | 257 | %%% custom header + footer 258 | \usepackage{fancyhdr} 259 | \fancypagestyle{plain}{% % overwrite plain pagestyle for entire document 260 | \fancyhf{} % clear all header and footer fields 261 | \fancyhead[L]{Tim Ebert} 262 | \fancyhead[R]{\ShortTitle} 263 | \fancyfoot[R]{\thepage} 264 | \renewcommand{\headrulewidth}{0.4pt} 265 | \renewcommand{\footrulewidth}{0.4pt}} 266 | \pagestyle{plain} 267 | 268 | % continue roman numbering in appendix 269 | % https://tex.stackexchange.com/questions/166416/continue-previous-roman-page-numbering-after-changing-to-arabic 270 | \newcounter{savepage} 271 | 272 | %%% custom numbering for requirements 273 | \newcounter{reqCounter} 274 | \newcommand{\requirement}{% 275 | \refstepcounter{reqCounter}% 276 | Req. \arabic{reqCounter}: 277 | } 278 | 279 | %%% spacing 280 | \usepackage{setspace} 281 | \onehalfspacing % spacing between lines 282 | 283 | \setlength{\parindent}{0pt} % paragraph indententation 284 | \setlength{\parskip}{0.8em} % spacing between paragraphs 285 | 286 | % % don't break page on new chapter 287 | % \usepackage{etoolbox} 288 | % \makeatletter 289 | % \patchcmd{\scr@startchapter}{\if@openright\cleardoublepage\else\clearpage\fi}{}{}{} 290 | % \makeatother 291 | 292 | %%% end header.text 293 | -------------------------------------------------------------------------------- /results/base/sharded-memory.csv: -------------------------------------------------------------------------------- 1 | ts,value,pod 2 | 1666596649,20037632,webhosting-operator-559494f768-7cjp8 3 | 1666596664,18804736,webhosting-operator-559494f768-7cjp8 4 | 1666596679,18804736,webhosting-operator-559494f768-7cjp8 5 | 1666596694,19480576,webhosting-operator-559494f768-7cjp8 6 | 1666596709,19480576,webhosting-operator-559494f768-7cjp8 7 | 1666596724,29515776,webhosting-operator-559494f768-7cjp8 8 | 1666596739,29515776,webhosting-operator-559494f768-7cjp8 9 | 1666596754,39096320,webhosting-operator-559494f768-7cjp8 10 | 1666596769,39096320,webhosting-operator-559494f768-7cjp8 11 | 1666596784,52047872,webhosting-operator-559494f768-7cjp8 12 | 1666596799,52047872,webhosting-operator-559494f768-7cjp8 13 | 1666596814,61652992,webhosting-operator-559494f768-7cjp8 14 | 1666596829,61652992,webhosting-operator-559494f768-7cjp8 15 | 1666596844,72740864,webhosting-operator-559494f768-7cjp8 16 | 1666596859,72740864,webhosting-operator-559494f768-7cjp8 17 | 1666596874,88399872,webhosting-operator-559494f768-7cjp8 18 | 1666596889,88399872,webhosting-operator-559494f768-7cjp8 19 | 1666596904,98484224,webhosting-operator-559494f768-7cjp8 20 | 1666596919,98484224,webhosting-operator-559494f768-7cjp8 21 | 1666596934,106823680,webhosting-operator-559494f768-7cjp8 22 | 1666596949,106823680,webhosting-operator-559494f768-7cjp8 23 | 1666596964,120811520,webhosting-operator-559494f768-7cjp8 24 | 1666596979,120811520,webhosting-operator-559494f768-7cjp8 25 | 1666596994,130035712,webhosting-operator-559494f768-7cjp8 26 | 1666597009,130035712,webhosting-operator-559494f768-7cjp8 27 | 1666597024,144015360,webhosting-operator-559494f768-7cjp8 28 | 1666597039,144015360,webhosting-operator-559494f768-7cjp8 29 | 1666597054,154730496,webhosting-operator-559494f768-7cjp8 30 | 1666597069,154730496,webhosting-operator-559494f768-7cjp8 31 | 1666597084,168763392,webhosting-operator-559494f768-7cjp8 32 | 1666597099,168763392,webhosting-operator-559494f768-7cjp8 33 | 1666597114,172560384,webhosting-operator-559494f768-7cjp8 34 | 1666597129,172560384,webhosting-operator-559494f768-7cjp8 35 | 1666597144,185790464,webhosting-operator-559494f768-7cjp8 36 | 1666597159,185790464,webhosting-operator-559494f768-7cjp8 37 | 1666597174,193748992,webhosting-operator-559494f768-7cjp8 38 | 1666597189,193748992,webhosting-operator-559494f768-7cjp8 39 | 1666597204,203444224,webhosting-operator-559494f768-7cjp8 40 | 1666597219,203444224,webhosting-operator-559494f768-7cjp8 41 | 1666597234,215756800,webhosting-operator-559494f768-7cjp8 42 | 1666597249,215756800,webhosting-operator-559494f768-7cjp8 43 | 1666597264,223580160,webhosting-operator-559494f768-7cjp8 44 | 1666597279,223580160,webhosting-operator-559494f768-7cjp8 45 | 1666597294,230436864,webhosting-operator-559494f768-7cjp8 46 | 1666597309,230436864,webhosting-operator-559494f768-7cjp8 47 | 1666597324,243703808,webhosting-operator-559494f768-7cjp8 48 | 1666597339,243703808,webhosting-operator-559494f768-7cjp8 49 | 1666597354,254754816,webhosting-operator-559494f768-7cjp8 50 | 1666597369,254754816,webhosting-operator-559494f768-7cjp8 51 | 1666597384,254754816,webhosting-operator-559494f768-7cjp8 52 | 1666597399,254754816,webhosting-operator-559494f768-7cjp8 53 | 1666597414,254754816,webhosting-operator-559494f768-7cjp8 54 | 1666597429,254754816,webhosting-operator-559494f768-7cjp8 55 | 1666596649,11436032,webhosting-operator-559494f768-rbsdv 56 | 1666596664,14970880,webhosting-operator-559494f768-rbsdv 57 | 1666596679,14970880,webhosting-operator-559494f768-rbsdv 58 | 1666596694,17088512,webhosting-operator-559494f768-rbsdv 59 | 1666596709,17088512,webhosting-operator-559494f768-rbsdv 60 | 1666596724,19251200,webhosting-operator-559494f768-rbsdv 61 | 1666596739,19251200,webhosting-operator-559494f768-rbsdv 62 | 1666596754,27549696,webhosting-operator-559494f768-rbsdv 63 | 1666596769,27549696,webhosting-operator-559494f768-rbsdv 64 | 1666596784,29982720,webhosting-operator-559494f768-rbsdv 65 | 1666596799,29982720,webhosting-operator-559494f768-rbsdv 66 | 1666596814,34136064,webhosting-operator-559494f768-rbsdv 67 | 1666596829,34136064,webhosting-operator-559494f768-rbsdv 68 | 1666596844,39034880,webhosting-operator-559494f768-rbsdv 69 | 1666596859,39034880,webhosting-operator-559494f768-rbsdv 70 | 1666596874,43655168,webhosting-operator-559494f768-rbsdv 71 | 1666596889,43655168,webhosting-operator-559494f768-rbsdv 72 | 1666596904,50511872,webhosting-operator-559494f768-rbsdv 73 | 1666596919,50511872,webhosting-operator-559494f768-rbsdv 74 | 1666596934,54308864,webhosting-operator-559494f768-rbsdv 75 | 1666596949,54308864,webhosting-operator-559494f768-rbsdv 76 | 1666596964,59195392,webhosting-operator-559494f768-rbsdv 77 | 1666596979,59195392,webhosting-operator-559494f768-rbsdv 78 | 1666596994,61558784,webhosting-operator-559494f768-rbsdv 79 | 1666597009,61558784,webhosting-operator-559494f768-rbsdv 80 | 1666597024,65953792,webhosting-operator-559494f768-rbsdv 81 | 1666597039,65953792,webhosting-operator-559494f768-rbsdv 82 | 1666597054,68132864,webhosting-operator-559494f768-rbsdv 83 | 1666597069,68132864,webhosting-operator-559494f768-rbsdv 84 | 1666597084,72859648,webhosting-operator-559494f768-rbsdv 85 | 1666597099,72859648,webhosting-operator-559494f768-rbsdv 86 | 1666597114,79491072,webhosting-operator-559494f768-rbsdv 87 | 1666597129,79491072,webhosting-operator-559494f768-rbsdv 88 | 1666597144,83771392,webhosting-operator-559494f768-rbsdv 89 | 1666597159,83771392,webhosting-operator-559494f768-rbsdv 90 | 1666597174,88379392,webhosting-operator-559494f768-rbsdv 91 | 1666597189,88379392,webhosting-operator-559494f768-rbsdv 92 | 1666597204,91189248,webhosting-operator-559494f768-rbsdv 93 | 1666597219,91189248,webhosting-operator-559494f768-rbsdv 94 | 1666597234,95604736,webhosting-operator-559494f768-rbsdv 95 | 1666597249,95604736,webhosting-operator-559494f768-rbsdv 96 | 1666597264,101486592,webhosting-operator-559494f768-rbsdv 97 | 1666597279,101486592,webhosting-operator-559494f768-rbsdv 98 | 1666597294,101756928,webhosting-operator-559494f768-rbsdv 99 | 1666597309,101756928,webhosting-operator-559494f768-rbsdv 100 | 1666597324,106254336,webhosting-operator-559494f768-rbsdv 101 | 1666597339,106254336,webhosting-operator-559494f768-rbsdv 102 | 1666597354,108388352,webhosting-operator-559494f768-rbsdv 103 | 1666597369,108388352,webhosting-operator-559494f768-rbsdv 104 | 1666597384,108388352,webhosting-operator-559494f768-rbsdv 105 | 1666597399,108388352,webhosting-operator-559494f768-rbsdv 106 | 1666597414,107036672,webhosting-operator-559494f768-rbsdv 107 | 1666597429,107036672,webhosting-operator-559494f768-rbsdv 108 | 1666596649,13381632,webhosting-operator-559494f768-sk4mv 109 | 1666596664,17715200,webhosting-operator-559494f768-sk4mv 110 | 1666596679,17715200,webhosting-operator-559494f768-sk4mv 111 | 1666596694,17715200,webhosting-operator-559494f768-sk4mv 112 | 1666596709,17715200,webhosting-operator-559494f768-sk4mv 113 | 1666596724,21073920,webhosting-operator-559494f768-sk4mv 114 | 1666596739,21073920,webhosting-operator-559494f768-sk4mv 115 | 1666596754,24969216,webhosting-operator-559494f768-sk4mv 116 | 1666596769,24969216,webhosting-operator-559494f768-sk4mv 117 | 1666596784,28848128,webhosting-operator-559494f768-sk4mv 118 | 1666596799,28848128,webhosting-operator-559494f768-sk4mv 119 | 1666596814,34619392,webhosting-operator-559494f768-sk4mv 120 | 1666596829,34619392,webhosting-operator-559494f768-sk4mv 121 | 1666596844,35966976,webhosting-operator-559494f768-sk4mv 122 | 1666596859,35966976,webhosting-operator-559494f768-sk4mv 123 | 1666596874,39272448,webhosting-operator-559494f768-sk4mv 124 | 1666596889,39272448,webhosting-operator-559494f768-sk4mv 125 | 1666596904,45498368,webhosting-operator-559494f768-sk4mv 126 | 1666596919,45498368,webhosting-operator-559494f768-sk4mv 127 | 1666596934,50266112,webhosting-operator-559494f768-sk4mv 128 | 1666596949,50266112,webhosting-operator-559494f768-sk4mv 129 | 1666596964,53395456,webhosting-operator-559494f768-sk4mv 130 | 1666596979,53395456,webhosting-operator-559494f768-sk4mv 131 | 1666596994,57278464,webhosting-operator-559494f768-sk4mv 132 | 1666597009,57278464,webhosting-operator-559494f768-sk4mv 133 | 1666597024,59469824,webhosting-operator-559494f768-sk4mv 134 | 1666597039,59469824,webhosting-operator-559494f768-sk4mv 135 | 1666597054,66142208,webhosting-operator-559494f768-sk4mv 136 | 1666597069,66142208,webhosting-operator-559494f768-sk4mv 137 | 1666597084,70488064,webhosting-operator-559494f768-sk4mv 138 | 1666597099,70488064,webhosting-operator-559494f768-sk4mv 139 | 1666597114,72110080,webhosting-operator-559494f768-sk4mv 140 | 1666597129,72110080,webhosting-operator-559494f768-sk4mv 141 | 1666597144,77295616,webhosting-operator-559494f768-sk4mv 142 | 1666597159,77295616,webhosting-operator-559494f768-sk4mv 143 | 1666597174,81833984,webhosting-operator-559494f768-sk4mv 144 | 1666597189,81833984,webhosting-operator-559494f768-sk4mv 145 | 1666597204,86343680,webhosting-operator-559494f768-sk4mv 146 | 1666597219,86343680,webhosting-operator-559494f768-sk4mv 147 | 1666597234,88457216,webhosting-operator-559494f768-sk4mv 148 | 1666597249,88457216,webhosting-operator-559494f768-sk4mv 149 | 1666597264,90591232,webhosting-operator-559494f768-sk4mv 150 | 1666597279,90591232,webhosting-operator-559494f768-sk4mv 151 | 1666597294,95105024,webhosting-operator-559494f768-sk4mv 152 | 1666597309,95105024,webhosting-operator-559494f768-sk4mv 153 | 1666597324,99373056,webhosting-operator-559494f768-sk4mv 154 | 1666597339,99373056,webhosting-operator-559494f768-sk4mv 155 | 1666597354,99508224,webhosting-operator-559494f768-sk4mv 156 | 1666597369,99508224,webhosting-operator-559494f768-sk4mv 157 | 1666597384,99508224,webhosting-operator-559494f768-sk4mv 158 | 1666597399,99508224,webhosting-operator-559494f768-sk4mv 159 | 1666597414,99102720,webhosting-operator-559494f768-sk4mv 160 | 1666597429,99102720,webhosting-operator-559494f768-sk4mv 161 | -------------------------------------------------------------------------------- /results/base/sharded-network_receive.csv: -------------------------------------------------------------------------------- 1 | ts,value,pod 2 | 1666596664,7696.766666666666,webhosting-operator-559494f768-7cjp8 3 | 1666596679,7696.766666666666,webhosting-operator-559494f768-7cjp8 4 | 1666596694,9766.6,webhosting-operator-559494f768-7cjp8 5 | 1666596709,9766.6,webhosting-operator-559494f768-7cjp8 6 | 1666596724,747328.6666666666,webhosting-operator-559494f768-7cjp8 7 | 1666596739,747328.6666666666,webhosting-operator-559494f768-7cjp8 8 | 1666596754,813851.3666666667,webhosting-operator-559494f768-7cjp8 9 | 1666596769,813851.3666666667,webhosting-operator-559494f768-7cjp8 10 | 1666596784,975903.2666666667,webhosting-operator-559494f768-7cjp8 11 | 1666596799,975903.2666666667,webhosting-operator-559494f768-7cjp8 12 | 1666596814,1483289.4,webhosting-operator-559494f768-7cjp8 13 | 1666596829,1483289.4,webhosting-operator-559494f768-7cjp8 14 | 1666596844,1051528.5666666667,webhosting-operator-559494f768-7cjp8 15 | 1666596859,1051528.5666666667,webhosting-operator-559494f768-7cjp8 16 | 1666596874,1300659.2,webhosting-operator-559494f768-7cjp8 17 | 1666596889,1300659.2,webhosting-operator-559494f768-7cjp8 18 | 1666596904,1175356.4,webhosting-operator-559494f768-7cjp8 19 | 1666596919,1175356.4,webhosting-operator-559494f768-7cjp8 20 | 1666596934,1463616.3666666667,webhosting-operator-559494f768-7cjp8 21 | 1666596949,1463616.3666666667,webhosting-operator-559494f768-7cjp8 22 | 1666596964,1693552.2,webhosting-operator-559494f768-7cjp8 23 | 1666596979,1693552.2,webhosting-operator-559494f768-7cjp8 24 | 1666596994,914709.1,webhosting-operator-559494f768-7cjp8 25 | 1666597009,914709.1,webhosting-operator-559494f768-7cjp8 26 | 1666597024,1844339.1333333333,webhosting-operator-559494f768-7cjp8 27 | 1666597039,1844339.1333333333,webhosting-operator-559494f768-7cjp8 28 | 1666597054,2319807.8666666667,webhosting-operator-559494f768-7cjp8 29 | 1666597069,2319807.8666666667,webhosting-operator-559494f768-7cjp8 30 | 1666597084,1688829.2666666666,webhosting-operator-559494f768-7cjp8 31 | 1666597099,1688829.2666666666,webhosting-operator-559494f768-7cjp8 32 | 1666597114,1537004.7,webhosting-operator-559494f768-7cjp8 33 | 1666597129,1537004.7,webhosting-operator-559494f768-7cjp8 34 | 1666597144,2190759.7666666666,webhosting-operator-559494f768-7cjp8 35 | 1666597159,2190759.7666666666,webhosting-operator-559494f768-7cjp8 36 | 1666597174,1575473,webhosting-operator-559494f768-7cjp8 37 | 1666597189,1575473,webhosting-operator-559494f768-7cjp8 38 | 1666597204,1596448.9,webhosting-operator-559494f768-7cjp8 39 | 1666597219,1596448.9,webhosting-operator-559494f768-7cjp8 40 | 1666597234,1143817.9,webhosting-operator-559494f768-7cjp8 41 | 1666597249,1143817.9,webhosting-operator-559494f768-7cjp8 42 | 1666597264,2311061.7333333334,webhosting-operator-559494f768-7cjp8 43 | 1666597279,2311061.7333333334,webhosting-operator-559494f768-7cjp8 44 | 1666597294,1789294.4333333333,webhosting-operator-559494f768-7cjp8 45 | 1666597309,1789294.4333333333,webhosting-operator-559494f768-7cjp8 46 | 1666597324,2347948.3,webhosting-operator-559494f768-7cjp8 47 | 1666597339,2347948.3,webhosting-operator-559494f768-7cjp8 48 | 1666597354,2284189.3,webhosting-operator-559494f768-7cjp8 49 | 1666597369,2284189.3,webhosting-operator-559494f768-7cjp8 50 | 1666597384,2577922.033333333,webhosting-operator-559494f768-7cjp8 51 | 1666597399,2577922.033333333,webhosting-operator-559494f768-7cjp8 52 | 1666597414,1241522,webhosting-operator-559494f768-7cjp8 53 | 1666597429,1241522,webhosting-operator-559494f768-7cjp8 54 | 1666596664,1027.0333333333333,webhosting-operator-559494f768-rbsdv 55 | 1666596679,1027.0333333333333,webhosting-operator-559494f768-rbsdv 56 | 1666596694,20114.966666666667,webhosting-operator-559494f768-rbsdv 57 | 1666596709,20114.966666666667,webhosting-operator-559494f768-rbsdv 58 | 1666596724,301821.8333333333,webhosting-operator-559494f768-rbsdv 59 | 1666596739,301821.8333333333,webhosting-operator-559494f768-rbsdv 60 | 1666596754,403914.63333333336,webhosting-operator-559494f768-rbsdv 61 | 1666596769,403914.63333333336,webhosting-operator-559494f768-rbsdv 62 | 1666596784,536388.5,webhosting-operator-559494f768-rbsdv 63 | 1666596799,536388.5,webhosting-operator-559494f768-rbsdv 64 | 1666596814,253055.5,webhosting-operator-559494f768-rbsdv 65 | 1666596829,253055.5,webhosting-operator-559494f768-rbsdv 66 | 1666596844,774640.0333333333,webhosting-operator-559494f768-rbsdv 67 | 1666596859,774640.0333333333,webhosting-operator-559494f768-rbsdv 68 | 1666596874,611477.3,webhosting-operator-559494f768-rbsdv 69 | 1666596889,611477.3,webhosting-operator-559494f768-rbsdv 70 | 1666596904,606630.4333333333,webhosting-operator-559494f768-rbsdv 71 | 1666596919,606630.4333333333,webhosting-operator-559494f768-rbsdv 72 | 1666596934,714420,webhosting-operator-559494f768-rbsdv 73 | 1666596949,714420,webhosting-operator-559494f768-rbsdv 74 | 1666596964,574572.5666666667,webhosting-operator-559494f768-rbsdv 75 | 1666596979,574572.5666666667,webhosting-operator-559494f768-rbsdv 76 | 1666596994,783198.3,webhosting-operator-559494f768-rbsdv 77 | 1666597009,783198.3,webhosting-operator-559494f768-rbsdv 78 | 1666597024,918328.3,webhosting-operator-559494f768-rbsdv 79 | 1666597039,918328.3,webhosting-operator-559494f768-rbsdv 80 | 1666597054,704373.4,webhosting-operator-559494f768-rbsdv 81 | 1666597069,704373.4,webhosting-operator-559494f768-rbsdv 82 | 1666597084,1080170.6666666667,webhosting-operator-559494f768-rbsdv 83 | 1666597099,1080170.6666666667,webhosting-operator-559494f768-rbsdv 84 | 1666597114,871013.4333333333,webhosting-operator-559494f768-rbsdv 85 | 1666597129,871013.4333333333,webhosting-operator-559494f768-rbsdv 86 | 1666597144,1038878.2333333333,webhosting-operator-559494f768-rbsdv 87 | 1666597159,1038878.2333333333,webhosting-operator-559494f768-rbsdv 88 | 1666597174,1036225.8,webhosting-operator-559494f768-rbsdv 89 | 1666597189,1036225.8,webhosting-operator-559494f768-rbsdv 90 | 1666597204,1079971.0333333334,webhosting-operator-559494f768-rbsdv 91 | 1666597219,1079971.0333333334,webhosting-operator-559494f768-rbsdv 92 | 1666597234,1002237.7,webhosting-operator-559494f768-rbsdv 93 | 1666597249,1002237.7,webhosting-operator-559494f768-rbsdv 94 | 1666597264,1101517.3666666667,webhosting-operator-559494f768-rbsdv 95 | 1666597279,1101517.3666666667,webhosting-operator-559494f768-rbsdv 96 | 1666597294,1350135,webhosting-operator-559494f768-rbsdv 97 | 1666597309,1350135,webhosting-operator-559494f768-rbsdv 98 | 1666597324,594864.8,webhosting-operator-559494f768-rbsdv 99 | 1666597339,594864.8,webhosting-operator-559494f768-rbsdv 100 | 1666597354,1039404.3,webhosting-operator-559494f768-rbsdv 101 | 1666597369,1039404.3,webhosting-operator-559494f768-rbsdv 102 | 1666597384,811162.3666666667,webhosting-operator-559494f768-rbsdv 103 | 1666597399,811162.3666666667,webhosting-operator-559494f768-rbsdv 104 | 1666597414,1036897.6666666666,webhosting-operator-559494f768-rbsdv 105 | 1666597429,1036897.6666666666,webhosting-operator-559494f768-rbsdv 106 | 1666596664,1472.9666666666667,webhosting-operator-559494f768-sk4mv 107 | 1666596679,1472.9666666666667,webhosting-operator-559494f768-sk4mv 108 | 1666596694,2268.766666666667,webhosting-operator-559494f768-sk4mv 109 | 1666596709,2268.766666666667,webhosting-operator-559494f768-sk4mv 110 | 1666596724,395115.86666666664,webhosting-operator-559494f768-sk4mv 111 | 1666596739,395115.86666666664,webhosting-operator-559494f768-sk4mv 112 | 1666596754,217069.1,webhosting-operator-559494f768-sk4mv 113 | 1666596769,217069.1,webhosting-operator-559494f768-sk4mv 114 | 1666596784,533214.9333333333,webhosting-operator-559494f768-sk4mv 115 | 1666596799,533214.9333333333,webhosting-operator-559494f768-sk4mv 116 | 1666596814,389164.76666666666,webhosting-operator-559494f768-sk4mv 117 | 1666596829,389164.76666666666,webhosting-operator-559494f768-sk4mv 118 | 1666596844,432291.7,webhosting-operator-559494f768-sk4mv 119 | 1666596859,432291.7,webhosting-operator-559494f768-sk4mv 120 | 1666596874,622157.0333333333,webhosting-operator-559494f768-sk4mv 121 | 1666596889,622157.0333333333,webhosting-operator-559494f768-sk4mv 122 | 1666596904,643900.4333333333,webhosting-operator-559494f768-sk4mv 123 | 1666596919,643900.4333333333,webhosting-operator-559494f768-sk4mv 124 | 1666596934,313061.4,webhosting-operator-559494f768-sk4mv 125 | 1666596949,313061.4,webhosting-operator-559494f768-sk4mv 126 | 1666596964,638661.4333333333,webhosting-operator-559494f768-sk4mv 127 | 1666596979,638661.4333333333,webhosting-operator-559494f768-sk4mv 128 | 1666596994,815696.2333333333,webhosting-operator-559494f768-sk4mv 129 | 1666597009,815696.2333333333,webhosting-operator-559494f768-sk4mv 130 | 1666597024,648336.8333333334,webhosting-operator-559494f768-sk4mv 131 | 1666597039,648336.8333333334,webhosting-operator-559494f768-sk4mv 132 | 1666597054,723433.9333333333,webhosting-operator-559494f768-sk4mv 133 | 1666597069,723433.9333333333,webhosting-operator-559494f768-sk4mv 134 | 1666597084,703435,webhosting-operator-559494f768-sk4mv 135 | 1666597099,703435,webhosting-operator-559494f768-sk4mv 136 | 1666597114,994627.1333333333,webhosting-operator-559494f768-sk4mv 137 | 1666597129,994627.1333333333,webhosting-operator-559494f768-sk4mv 138 | 1666597144,591473.4,webhosting-operator-559494f768-sk4mv 139 | 1666597159,591473.4,webhosting-operator-559494f768-sk4mv 140 | 1666597174,901744.5333333333,webhosting-operator-559494f768-sk4mv 141 | 1666597189,901744.5333333333,webhosting-operator-559494f768-sk4mv 142 | 1666597204,877303.3666666667,webhosting-operator-559494f768-sk4mv 143 | 1666597219,877303.3666666667,webhosting-operator-559494f768-sk4mv 144 | 1666597234,886034.1,webhosting-operator-559494f768-sk4mv 145 | 1666597249,886034.1,webhosting-operator-559494f768-sk4mv 146 | 1666597264,1022744.6333333333,webhosting-operator-559494f768-sk4mv 147 | 1666597279,1022744.6333333333,webhosting-operator-559494f768-sk4mv 148 | 1666597294,957067.4333333333,webhosting-operator-559494f768-sk4mv 149 | 1666597309,957067.4333333333,webhosting-operator-559494f768-sk4mv 150 | 1666597324,1094316.6,webhosting-operator-559494f768-sk4mv 151 | 1666597339,1094316.6,webhosting-operator-559494f768-sk4mv 152 | 1666597354,978571.1333333333,webhosting-operator-559494f768-sk4mv 153 | 1666597369,978571.1333333333,webhosting-operator-559494f768-sk4mv 154 | 1666597384,539548.3666666667,webhosting-operator-559494f768-sk4mv 155 | 1666597399,539548.3666666667,webhosting-operator-559494f768-sk4mv 156 | 1666597414,814772.9666666667,webhosting-operator-559494f768-sk4mv 157 | 1666597429,814772.9666666667,webhosting-operator-559494f768-sk4mv 158 | -------------------------------------------------------------------------------- /results/base/sharded-network_transmit.csv: -------------------------------------------------------------------------------- 1 | ts,value,pod 2 | 1666596664,2625.0333333333333,webhosting-operator-559494f768-7cjp8 3 | 1666596679,2625.0333333333333,webhosting-operator-559494f768-7cjp8 4 | 1666596694,2556.5,webhosting-operator-559494f768-7cjp8 5 | 1666596709,2556.5,webhosting-operator-559494f768-7cjp8 6 | 1666596724,196336.03333333333,webhosting-operator-559494f768-7cjp8 7 | 1666596739,196336.03333333333,webhosting-operator-559494f768-7cjp8 8 | 1666596754,211048.7,webhosting-operator-559494f768-7cjp8 9 | 1666596769,211048.7,webhosting-operator-559494f768-7cjp8 10 | 1666596784,257675.16666666666,webhosting-operator-559494f768-7cjp8 11 | 1666596799,257675.16666666666,webhosting-operator-559494f768-7cjp8 12 | 1666596814,379060.36666666664,webhosting-operator-559494f768-7cjp8 13 | 1666596829,379060.36666666664,webhosting-operator-559494f768-7cjp8 14 | 1666596844,257222.3,webhosting-operator-559494f768-7cjp8 15 | 1666596859,257222.3,webhosting-operator-559494f768-7cjp8 16 | 1666596874,331094.6666666667,webhosting-operator-559494f768-7cjp8 17 | 1666596889,331094.6666666667,webhosting-operator-559494f768-7cjp8 18 | 1666596904,299968.7,webhosting-operator-559494f768-7cjp8 19 | 1666596919,299968.7,webhosting-operator-559494f768-7cjp8 20 | 1666596934,367666.06666666665,webhosting-operator-559494f768-7cjp8 21 | 1666596949,367666.06666666665,webhosting-operator-559494f768-7cjp8 22 | 1666596964,410057.3333333333,webhosting-operator-559494f768-7cjp8 23 | 1666596979,410057.3333333333,webhosting-operator-559494f768-7cjp8 24 | 1666596994,225064.13333333333,webhosting-operator-559494f768-7cjp8 25 | 1666597009,225064.13333333333,webhosting-operator-559494f768-7cjp8 26 | 1666597024,446845.86666666664,webhosting-operator-559494f768-7cjp8 27 | 1666597039,446845.86666666664,webhosting-operator-559494f768-7cjp8 28 | 1666597054,557145.2333333333,webhosting-operator-559494f768-7cjp8 29 | 1666597069,557145.2333333333,webhosting-operator-559494f768-7cjp8 30 | 1666597084,390075.63333333336,webhosting-operator-559494f768-7cjp8 31 | 1666597099,390075.63333333336,webhosting-operator-559494f768-7cjp8 32 | 1666597114,338815.76666666666,webhosting-operator-559494f768-7cjp8 33 | 1666597129,338815.76666666666,webhosting-operator-559494f768-7cjp8 34 | 1666597144,523814.1666666667,webhosting-operator-559494f768-7cjp8 35 | 1666597159,523814.1666666667,webhosting-operator-559494f768-7cjp8 36 | 1666597174,280235.36666666664,webhosting-operator-559494f768-7cjp8 37 | 1666597189,280235.36666666664,webhosting-operator-559494f768-7cjp8 38 | 1666597204,292919.73333333334,webhosting-operator-559494f768-7cjp8 39 | 1666597219,292919.73333333334,webhosting-operator-559494f768-7cjp8 40 | 1666597234,230730.53333333333,webhosting-operator-559494f768-7cjp8 41 | 1666597249,230730.53333333333,webhosting-operator-559494f768-7cjp8 42 | 1666597264,456943.56666666665,webhosting-operator-559494f768-7cjp8 43 | 1666597279,456943.56666666665,webhosting-operator-559494f768-7cjp8 44 | 1666597294,358326.73333333334,webhosting-operator-559494f768-7cjp8 45 | 1666597309,358326.73333333334,webhosting-operator-559494f768-7cjp8 46 | 1666597324,501301.36666666664,webhosting-operator-559494f768-7cjp8 47 | 1666597339,501301.36666666664,webhosting-operator-559494f768-7cjp8 48 | 1666597354,556807.3666666667,webhosting-operator-559494f768-7cjp8 49 | 1666597369,556807.3666666667,webhosting-operator-559494f768-7cjp8 50 | 1666597384,676847.8,webhosting-operator-559494f768-7cjp8 51 | 1666597399,676847.8,webhosting-operator-559494f768-7cjp8 52 | 1666597414,323115.86666666664,webhosting-operator-559494f768-7cjp8 53 | 1666597429,323115.86666666664,webhosting-operator-559494f768-7cjp8 54 | 1666596664,982.9333333333333,webhosting-operator-559494f768-rbsdv 55 | 1666596679,982.9333333333333,webhosting-operator-559494f768-rbsdv 56 | 1666596694,10180,webhosting-operator-559494f768-rbsdv 57 | 1666596709,10180,webhosting-operator-559494f768-rbsdv 58 | 1666596724,131654.53333333333,webhosting-operator-559494f768-rbsdv 59 | 1666596739,131654.53333333333,webhosting-operator-559494f768-rbsdv 60 | 1666596754,173658.43333333332,webhosting-operator-559494f768-rbsdv 61 | 1666596769,173658.43333333332,webhosting-operator-559494f768-rbsdv 62 | 1666596784,231620.86666666667,webhosting-operator-559494f768-rbsdv 63 | 1666596799,231620.86666666667,webhosting-operator-559494f768-rbsdv 64 | 1666596814,108574,webhosting-operator-559494f768-rbsdv 65 | 1666596829,108574,webhosting-operator-559494f768-rbsdv 66 | 1666596844,335767.56666666665,webhosting-operator-559494f768-rbsdv 67 | 1666596859,335767.56666666665,webhosting-operator-559494f768-rbsdv 68 | 1666596874,264026.26666666666,webhosting-operator-559494f768-rbsdv 69 | 1666596889,264026.26666666666,webhosting-operator-559494f768-rbsdv 70 | 1666596904,261420.66666666666,webhosting-operator-559494f768-rbsdv 71 | 1666596919,261420.66666666666,webhosting-operator-559494f768-rbsdv 72 | 1666596934,307755.56666666665,webhosting-operator-559494f768-rbsdv 73 | 1666596949,307755.56666666665,webhosting-operator-559494f768-rbsdv 74 | 1666596964,245840.5,webhosting-operator-559494f768-rbsdv 75 | 1666596979,245840.5,webhosting-operator-559494f768-rbsdv 76 | 1666596994,336284.26666666666,webhosting-operator-559494f768-rbsdv 77 | 1666597009,336284.26666666666,webhosting-operator-559494f768-rbsdv 78 | 1666597024,390360.63333333336,webhosting-operator-559494f768-rbsdv 79 | 1666597039,390360.63333333336,webhosting-operator-559494f768-rbsdv 80 | 1666597054,299364.76666666666,webhosting-operator-559494f768-rbsdv 81 | 1666597069,299364.76666666666,webhosting-operator-559494f768-rbsdv 82 | 1666597084,454691.4666666667,webhosting-operator-559494f768-rbsdv 83 | 1666597099,454691.4666666667,webhosting-operator-559494f768-rbsdv 84 | 1666597114,367329.8333333333,webhosting-operator-559494f768-rbsdv 85 | 1666597129,367329.8333333333,webhosting-operator-559494f768-rbsdv 86 | 1666597144,447029.8,webhosting-operator-559494f768-rbsdv 87 | 1666597159,447029.8,webhosting-operator-559494f768-rbsdv 88 | 1666597174,410735.13333333336,webhosting-operator-559494f768-rbsdv 89 | 1666597189,410735.13333333336,webhosting-operator-559494f768-rbsdv 90 | 1666597204,422337.63333333336,webhosting-operator-559494f768-rbsdv 91 | 1666597219,422337.63333333336,webhosting-operator-559494f768-rbsdv 92 | 1666597234,413722.3333333333,webhosting-operator-559494f768-rbsdv 93 | 1666597249,413722.3333333333,webhosting-operator-559494f768-rbsdv 94 | 1666597264,439002.86666666664,webhosting-operator-559494f768-rbsdv 95 | 1666597279,439002.86666666664,webhosting-operator-559494f768-rbsdv 96 | 1666597294,547497.7,webhosting-operator-559494f768-rbsdv 97 | 1666597309,547497.7,webhosting-operator-559494f768-rbsdv 98 | 1666597324,241589.13333333333,webhosting-operator-559494f768-rbsdv 99 | 1666597339,241589.13333333333,webhosting-operator-559494f768-rbsdv 100 | 1666597354,425071.6666666667,webhosting-operator-559494f768-rbsdv 101 | 1666597369,425071.6666666667,webhosting-operator-559494f768-rbsdv 102 | 1666597384,332684.7,webhosting-operator-559494f768-rbsdv 103 | 1666597399,332684.7,webhosting-operator-559494f768-rbsdv 104 | 1666597414,423171.26666666666,webhosting-operator-559494f768-rbsdv 105 | 1666597429,423171.26666666666,webhosting-operator-559494f768-rbsdv 106 | 1666596664,1441.3666666666666,webhosting-operator-559494f768-sk4mv 107 | 1666596679,1441.3666666666666,webhosting-operator-559494f768-sk4mv 108 | 1666596694,937.2666666666667,webhosting-operator-559494f768-sk4mv 109 | 1666596709,937.2666666666667,webhosting-operator-559494f768-sk4mv 110 | 1666596724,173398.13333333333,webhosting-operator-559494f768-sk4mv 111 | 1666596739,173398.13333333333,webhosting-operator-559494f768-sk4mv 112 | 1666596754,93325.7,webhosting-operator-559494f768-sk4mv 113 | 1666596769,93325.7,webhosting-operator-559494f768-sk4mv 114 | 1666596784,230819.73333333334,webhosting-operator-559494f768-sk4mv 115 | 1666596799,230819.73333333334,webhosting-operator-559494f768-sk4mv 116 | 1666596814,167495.73333333334,webhosting-operator-559494f768-sk4mv 117 | 1666596829,167495.73333333334,webhosting-operator-559494f768-sk4mv 118 | 1666596844,185381.76666666666,webhosting-operator-559494f768-sk4mv 119 | 1666596859,185381.76666666666,webhosting-operator-559494f768-sk4mv 120 | 1666596874,270189.23333333334,webhosting-operator-559494f768-sk4mv 121 | 1666596889,270189.23333333334,webhosting-operator-559494f768-sk4mv 122 | 1666596904,277432.4666666667,webhosting-operator-559494f768-sk4mv 123 | 1666596919,277432.4666666667,webhosting-operator-559494f768-sk4mv 124 | 1666596934,134071.6,webhosting-operator-559494f768-sk4mv 125 | 1666596949,134071.6,webhosting-operator-559494f768-sk4mv 126 | 1666596964,271658.9,webhosting-operator-559494f768-sk4mv 127 | 1666596979,271658.9,webhosting-operator-559494f768-sk4mv 128 | 1666596994,350681.6,webhosting-operator-559494f768-sk4mv 129 | 1666597009,350681.6,webhosting-operator-559494f768-sk4mv 130 | 1666597024,275744.73333333334,webhosting-operator-559494f768-sk4mv 131 | 1666597039,275744.73333333334,webhosting-operator-559494f768-sk4mv 132 | 1666597054,307175.86666666664,webhosting-operator-559494f768-sk4mv 133 | 1666597069,307175.86666666664,webhosting-operator-559494f768-sk4mv 134 | 1666597084,294621.86666666664,webhosting-operator-559494f768-sk4mv 135 | 1666597099,294621.86666666664,webhosting-operator-559494f768-sk4mv 136 | 1666597114,413432.6,webhosting-operator-559494f768-sk4mv 137 | 1666597129,413432.6,webhosting-operator-559494f768-sk4mv 138 | 1666597144,268207.93333333335,webhosting-operator-559494f768-sk4mv 139 | 1666597159,268207.93333333335,webhosting-operator-559494f768-sk4mv 140 | 1666597174,337912.3,webhosting-operator-559494f768-sk4mv 141 | 1666597189,337912.3,webhosting-operator-559494f768-sk4mv 142 | 1666597204,331168.0333333333,webhosting-operator-559494f768-sk4mv 143 | 1666597219,331168.0333333333,webhosting-operator-559494f768-sk4mv 144 | 1666597234,343941.86666666664,webhosting-operator-559494f768-sk4mv 145 | 1666597249,343941.86666666664,webhosting-operator-559494f768-sk4mv 146 | 1666597264,390333.5,webhosting-operator-559494f768-sk4mv 147 | 1666597279,390333.5,webhosting-operator-559494f768-sk4mv 148 | 1666597294,370201.43333333335,webhosting-operator-559494f768-sk4mv 149 | 1666597309,370201.43333333335,webhosting-operator-559494f768-sk4mv 150 | 1666597324,442618.9666666667,webhosting-operator-559494f768-sk4mv 151 | 1666597339,442618.9666666667,webhosting-operator-559494f768-sk4mv 152 | 1666597354,404429.56666666665,webhosting-operator-559494f768-sk4mv 153 | 1666597369,404429.56666666665,webhosting-operator-559494f768-sk4mv 154 | 1666597384,221049.73333333334,webhosting-operator-559494f768-sk4mv 155 | 1666597399,221049.73333333334,webhosting-operator-559494f768-sk4mv 156 | 1666597414,334666.13333333336,webhosting-operator-559494f768-sk4mv 157 | 1666597429,334666.13333333336,webhosting-operator-559494f768-sk4mv 158 | -------------------------------------------------------------------------------- /results/base/sharded-cpu.csv: -------------------------------------------------------------------------------- 1 | ts,value,pod 2 | 1666596664,0.006870158596395834,webhosting-operator-559494f768-7cjp8 3 | 1666596679,0.008343365783895834,webhosting-operator-559494f768-7cjp8 4 | 1666596694,0.009309393440241669,webhosting-operator-559494f768-7cjp8 5 | 1666596709,0.010676093565241667,webhosting-operator-559494f768-7cjp8 6 | 1666596724,0.06585568157654288,webhosting-operator-559494f768-7cjp8 7 | 1666596739,0.07459928385555556,webhosting-operator-559494f768-7cjp8 8 | 1666596754,0.14839399381132304,webhosting-operator-559494f768-7cjp8 9 | 1666596769,0.16876394823333332,webhosting-operator-559494f768-7cjp8 10 | 1666596784,0.23661774029381058,webhosting-operator-559494f768-7cjp8 11 | 1666596799,0.2701590447333333,webhosting-operator-559494f768-7cjp8 12 | 1666596814,0.32208721037777777,webhosting-operator-559494f768-7cjp8 13 | 1666596829,0.32208721037777777,webhosting-operator-559494f768-7cjp8 14 | 1666596844,0.3796983615333333,webhosting-operator-559494f768-7cjp8 15 | 1666596859,0.3796983615333333,webhosting-operator-559494f768-7cjp8 16 | 1666596874,0.4492864225666666,webhosting-operator-559494f768-7cjp8 17 | 1666596889,0.4492864225666666,webhosting-operator-559494f768-7cjp8 18 | 1666596904,0.44353327237777784,webhosting-operator-559494f768-7cjp8 19 | 1666596919,0.44353327237777784,webhosting-operator-559494f768-7cjp8 20 | 1666596934,0.41348423195555556,webhosting-operator-559494f768-7cjp8 21 | 1666596949,0.41348423195555556,webhosting-operator-559494f768-7cjp8 22 | 1666596964,0.3482667227888888,webhosting-operator-559494f768-7cjp8 23 | 1666596979,0.3482667227888888,webhosting-operator-559494f768-7cjp8 24 | 1666596994,0.3975502792333333,webhosting-operator-559494f768-7cjp8 25 | 1666597009,0.3975502792333333,webhosting-operator-559494f768-7cjp8 26 | 1666597024,0.4373904885555556,webhosting-operator-559494f768-7cjp8 27 | 1666597039,0.4373904885555556,webhosting-operator-559494f768-7cjp8 28 | 1666597054,0.443588372611111,webhosting-operator-559494f768-7cjp8 29 | 1666597069,0.443588372611111,webhosting-operator-559494f768-7cjp8 30 | 1666597084,0.4381407074444442,webhosting-operator-559494f768-7cjp8 31 | 1666597099,0.4381407074444442,webhosting-operator-559494f768-7cjp8 32 | 1666597114,0.39699992203333334,webhosting-operator-559494f768-7cjp8 33 | 1666597129,0.39699992203333334,webhosting-operator-559494f768-7cjp8 34 | 1666597144,0.4358491316777778,webhosting-operator-559494f768-7cjp8 35 | 1666597159,0.4358491316777778,webhosting-operator-559494f768-7cjp8 36 | 1666597174,0.38779345073333343,webhosting-operator-559494f768-7cjp8 37 | 1666597189,0.38779345073333343,webhosting-operator-559494f768-7cjp8 38 | 1666597204,0.4139561933666666,webhosting-operator-559494f768-7cjp8 39 | 1666597219,0.4139561933666666,webhosting-operator-559494f768-7cjp8 40 | 1666597234,0.3820271940555557,webhosting-operator-559494f768-7cjp8 41 | 1666597249,0.3820271940555557,webhosting-operator-559494f768-7cjp8 42 | 1666597264,0.4423649502666668,webhosting-operator-559494f768-7cjp8 43 | 1666597279,0.4423649502666668,webhosting-operator-559494f768-7cjp8 44 | 1666597294,0.3919834999555553,webhosting-operator-559494f768-7cjp8 45 | 1666597309,0.3919834999555553,webhosting-operator-559494f768-7cjp8 46 | 1666597324,0.4635802145111111,webhosting-operator-559494f768-7cjp8 47 | 1666597339,0.4635802145111111,webhosting-operator-559494f768-7cjp8 48 | 1666597354,0.44912767386666663,webhosting-operator-559494f768-7cjp8 49 | 1666597369,0.44912767386666663,webhosting-operator-559494f768-7cjp8 50 | 1666597384,0.5242043067333333,webhosting-operator-559494f768-7cjp8 51 | 1666597399,0.5242043067333333,webhosting-operator-559494f768-7cjp8 52 | 1666597414,0.45185209277777766,webhosting-operator-559494f768-7cjp8 53 | 1666597429,0.45185209277777766,webhosting-operator-559494f768-7cjp8 54 | 1666596664,0.001705554455463889,webhosting-operator-559494f768-rbsdv 55 | 1666596679,0.002158926247130556,webhosting-operator-559494f768-rbsdv 56 | 1666596694,0.007672440535362501,webhosting-operator-559494f768-rbsdv 57 | 1666596709,0.0089320532228625,webhosting-operator-559494f768-rbsdv 58 | 1666596724,0.02477885217229083,webhosting-operator-559494f768-rbsdv 59 | 1666596739,0.0273349081,webhosting-operator-559494f768-rbsdv 60 | 1666596754,0.0757710328628348,webhosting-operator-559494f768-rbsdv 61 | 1666596769,0.08659066782222222,webhosting-operator-559494f768-rbsdv 62 | 1666596784,0.12166928968239918,webhosting-operator-559494f768-rbsdv 63 | 1666596799,0.13600440509999998,webhosting-operator-559494f768-rbsdv 64 | 1666596814,0.14907535084590592,webhosting-operator-559494f768-rbsdv 65 | 1666596829,0.15013448328888887,webhosting-operator-559494f768-rbsdv 66 | 1666596844,0.1930304921,webhosting-operator-559494f768-rbsdv 67 | 1666596859,0.1930304921,webhosting-operator-559494f768-rbsdv 68 | 1666596874,0.20422917331111112,webhosting-operator-559494f768-rbsdv 69 | 1666596889,0.20422917331111112,webhosting-operator-559494f768-rbsdv 70 | 1666596904,0.2617348291888889,webhosting-operator-559494f768-rbsdv 71 | 1666596919,0.2617348291888889,webhosting-operator-559494f768-rbsdv 72 | 1666596934,0.23349902745555554,webhosting-operator-559494f768-rbsdv 73 | 1666596949,0.23349902745555554,webhosting-operator-559494f768-rbsdv 74 | 1666596964,0.23246061894444445,webhosting-operator-559494f768-rbsdv 75 | 1666596979,0.23246061894444445,webhosting-operator-559494f768-rbsdv 76 | 1666596994,0.21269405918888884,webhosting-operator-559494f768-rbsdv 77 | 1666597009,0.21269405918888884,webhosting-operator-559494f768-rbsdv 78 | 1666597024,0.2278223930666667,webhosting-operator-559494f768-rbsdv 79 | 1666597039,0.2278223930666667,webhosting-operator-559494f768-rbsdv 80 | 1666597054,0.21085173241111105,webhosting-operator-559494f768-rbsdv 81 | 1666597069,0.21085173241111105,webhosting-operator-559494f768-rbsdv 82 | 1666597084,0.23161592936666667,webhosting-operator-559494f768-rbsdv 83 | 1666597099,0.23161592936666667,webhosting-operator-559494f768-rbsdv 84 | 1666597114,0.23415208267777768,webhosting-operator-559494f768-rbsdv 85 | 1666597129,0.23415208267777768,webhosting-operator-559494f768-rbsdv 86 | 1666597144,0.29392729235555554,webhosting-operator-559494f768-rbsdv 87 | 1666597159,0.29392729235555554,webhosting-operator-559494f768-rbsdv 88 | 1666597174,0.28088234209999996,webhosting-operator-559494f768-rbsdv 89 | 1666597189,0.28088234209999996,webhosting-operator-559494f768-rbsdv 90 | 1666597204,0.31248783104444455,webhosting-operator-559494f768-rbsdv 91 | 1666597219,0.31248783104444455,webhosting-operator-559494f768-rbsdv 92 | 1666597234,0.3038560810222222,webhosting-operator-559494f768-rbsdv 93 | 1666597249,0.3038560810222222,webhosting-operator-559494f768-rbsdv 94 | 1666597264,0.31327967456666683,webhosting-operator-559494f768-rbsdv 95 | 1666597279,0.31327967456666683,webhosting-operator-559494f768-rbsdv 96 | 1666597294,0.2987820713333331,webhosting-operator-559494f768-rbsdv 97 | 1666597309,0.2987820713333331,webhosting-operator-559494f768-rbsdv 98 | 1666597324,0.2792227360333333,webhosting-operator-559494f768-rbsdv 99 | 1666597339,0.2792227360333333,webhosting-operator-559494f768-rbsdv 100 | 1666597354,0.2926672589222221,webhosting-operator-559494f768-rbsdv 101 | 1666597369,0.2926672589222221,webhosting-operator-559494f768-rbsdv 102 | 1666597384,0.23001198772222248,webhosting-operator-559494f768-rbsdv 103 | 1666597399,0.23001198772222248,webhosting-operator-559494f768-rbsdv 104 | 1666597414,0.259353387577778,webhosting-operator-559494f768-rbsdv 105 | 1666597429,0.259353387577778,webhosting-operator-559494f768-rbsdv 106 | 1666596664,0.0010036076655327779,webhosting-operator-559494f768-sk4mv 107 | 1666596679,0.0012703874238661113,webhosting-operator-559494f768-sk4mv 108 | 1666596694,0.0022748177981870834,webhosting-operator-559494f768-sk4mv 109 | 1666596709,0.002669618929437083,webhosting-operator-559494f768-sk4mv 110 | 1666596724,0.03064178677429027,webhosting-operator-559494f768-sk4mv 111 | 1666596739,0.03449669016666666,webhosting-operator-559494f768-sk4mv 112 | 1666596754,0.06043129029876611,webhosting-operator-559494f768-sk4mv 113 | 1666596769,0.06906994155709945,webhosting-operator-559494f768-sk4mv 114 | 1666596784,0.11082277198131453,webhosting-operator-559494f768-sk4mv 115 | 1666596799,0.126759016138259,webhosting-operator-559494f768-sk4mv 116 | 1666596814,0.15313995290000001,webhosting-operator-559494f768-sk4mv 117 | 1666596829,0.15313995290000001,webhosting-operator-559494f768-sk4mv 118 | 1666596844,0.18213317717777774,webhosting-operator-559494f768-sk4mv 119 | 1666596859,0.18213317717777774,webhosting-operator-559494f768-sk4mv 120 | 1666596874,0.17802293012222223,webhosting-operator-559494f768-sk4mv 121 | 1666596889,0.17802293012222223,webhosting-operator-559494f768-sk4mv 122 | 1666596904,0.1844057143111111,webhosting-operator-559494f768-sk4mv 123 | 1666596919,0.1844057143111111,webhosting-operator-559494f768-sk4mv 124 | 1666596934,0.18924686227777776,webhosting-operator-559494f768-sk4mv 125 | 1666596949,0.18924686227777776,webhosting-operator-559494f768-sk4mv 126 | 1666596964,0.21336269073333336,webhosting-operator-559494f768-sk4mv 127 | 1666596979,0.21336269073333336,webhosting-operator-559494f768-sk4mv 128 | 1666596994,0.20542823746666666,webhosting-operator-559494f768-sk4mv 129 | 1666597009,0.20542823746666666,webhosting-operator-559494f768-sk4mv 130 | 1666597024,0.18103580148888893,webhosting-operator-559494f768-sk4mv 131 | 1666597039,0.18103580148888893,webhosting-operator-559494f768-sk4mv 132 | 1666597054,0.21643635425555557,webhosting-operator-559494f768-sk4mv 133 | 1666597069,0.21643635425555557,webhosting-operator-559494f768-sk4mv 134 | 1666597084,0.23188734786666662,webhosting-operator-559494f768-sk4mv 135 | 1666597099,0.23188734786666662,webhosting-operator-559494f768-sk4mv 136 | 1666597114,0.25628986077777777,webhosting-operator-559494f768-sk4mv 137 | 1666597129,0.25628986077777777,webhosting-operator-559494f768-sk4mv 138 | 1666597144,0.18942188982222222,webhosting-operator-559494f768-sk4mv 139 | 1666597159,0.18942188982222222,webhosting-operator-559494f768-sk4mv 140 | 1666597174,0.19274385294444443,webhosting-operator-559494f768-sk4mv 141 | 1666597189,0.19274385294444443,webhosting-operator-559494f768-sk4mv 142 | 1666597204,0.23351212691111112,webhosting-operator-559494f768-sk4mv 143 | 1666597219,0.23351212691111112,webhosting-operator-559494f768-sk4mv 144 | 1666597234,0.25903099346666664,webhosting-operator-559494f768-sk4mv 145 | 1666597249,0.25903099346666664,webhosting-operator-559494f768-sk4mv 146 | 1666597264,0.25025304422222217,webhosting-operator-559494f768-sk4mv 147 | 1666597279,0.25025304422222217,webhosting-operator-559494f768-sk4mv 148 | 1666597294,0.2249525712111111,webhosting-operator-559494f768-sk4mv 149 | 1666597309,0.2249525712111111,webhosting-operator-559494f768-sk4mv 150 | 1666597324,0.23866715294444432,webhosting-operator-559494f768-sk4mv 151 | 1666597339,0.23866715294444432,webhosting-operator-559494f768-sk4mv 152 | 1666597354,0.2526146542111112,webhosting-operator-559494f768-sk4mv 153 | 1666597369,0.2526146542111112,webhosting-operator-559494f768-sk4mv 154 | 1666597384,0.24046059001111106,webhosting-operator-559494f768-sk4mv 155 | 1666597399,0.24046059001111106,webhosting-operator-559494f768-sk4mv 156 | 1666597414,0.22588872830000006,webhosting-operator-559494f768-sk4mv 157 | 1666597429,0.22588872830000006,webhosting-operator-559494f768-sk4mv 158 | -------------------------------------------------------------------------------- /content/bibliography.bib: -------------------------------------------------------------------------------- 1 | @misc{k8sdocs, 2 | title = {Kubernetes Documentation}, 3 | author = {{The Kubernetes Authors}}, 4 | date = {2022}, 5 | url = {https://kubernetes.io/docs/}, 6 | urldate = {2022-06-28}, 7 | } 8 | 9 | @misc{k8s, 10 | title = {Kubernetes}, 11 | author = {{The Kubernetes Authors}}, 12 | date = {2022}, 13 | url = {https://github.com/kubernetes/kubernetes}, 14 | urldate = {2022-06-28}, 15 | } 16 | 17 | @misc{promdocs, 18 | title = {Prometheus Documentation}, 19 | author = {{Prometheus Authors}}, 20 | date = {2022}, 21 | url = {https://prometheus.io/docs/}, 22 | urldate = {2022-10-25}, 23 | } 24 | 25 | @misc{grafanadocs, 26 | title = {Grafana Documentation}, 27 | author = {{Grafana Labs}}, 28 | date = {2022}, 29 | url = {https://grafana.com/docs/grafana/latest/}, 30 | urldate = {2022-10-25}, 31 | } 32 | 33 | @misc{etcddocs, 34 | title = {etcd Documentation}, 35 | author = {{The etcd Authors}}, 36 | date = {2022}, 37 | url = {https://etcd.io/docs/v3.5/}, 38 | urldate = {2022-07-03}, 39 | } 40 | 41 | @phdthesis{fielding2000architectural, 42 | title = {Architectural styles and the design of network-based software architectures}, 43 | author = {Fielding, Roy Thomas}, 44 | year = {2000}, 45 | school = {University of California, Irvine} 46 | } 47 | 48 | @misc{k8sdesign, 49 | title = {Kubernetes Design Proposals Archive}, 50 | author = {{The Kubernetes Authors}}, 51 | date = {2022}, 52 | url = {https://github.com/kubernetes/design-proposals-archive}, 53 | urldate = {2022-07-04}, 54 | } 55 | 56 | @misc{k8scommunity, 57 | title = {Kubernetes Community}, 58 | author = {{The Kubernetes Authors}}, 59 | date = {2022}, 60 | url = {https://github.com/kubernetes/community}, 61 | urldate = {2022-07-05}, 62 | } 63 | 64 | @misc{samplecontroller, 65 | title = {Kubernetes Sample Controller}, 66 | author = {{The Kubernetes Authors}}, 67 | date = {2022}, 68 | url = {https://github.com/kubernetes/sample-controller}, 69 | urldate = {2022-07-06}, 70 | } 71 | 72 | @inproceedings{karger1997consistent, 73 | author = {Karger, David and Lehman, Eric and Leighton, Tom and Panigrahy, Rina and Levine, Matthew and Lewin, Daniel}, 74 | title = {Consistent Hashing and Random Trees: Distributed Caching Protocols for Relieving Hot Spots on the World Wide Web}, 75 | year = {1997}, 76 | isbn = {0897918886}, 77 | publisher = {Association for Computing Machinery}, 78 | address = {New York, NY, USA}, 79 | booktitle = {Proceedings of the Twenty-Ninth Annual ACM Symposium on Theory of Computing}, 80 | pages = {654–663}, 81 | numpages = {10}, 82 | location = {El Paso, Texas, USA}, 83 | series = {STOC '97} 84 | } 85 | 86 | @inproceedings{stoica2001chord, 87 | author = {Stoica, Ion and Morris, Robert and Karger, David and Kaashoek, M. Frans and Balakrishnan, Hari}, 88 | title = {Chord: A Scalable Peer-to-Peer Lookup Service for Internet Applications}, 89 | year = {2001}, 90 | isbn = {1581134118}, 91 | publisher = {Association for Computing Machinery}, 92 | address = {New York, NY, USA}, 93 | abstract = {A fundamental problem that confronts peer-to-peer applications is to efficiently locate the node that stores a particular data item. This paper presents Chord, a distributed lookup protocol that addresses this problem. Chord provides support for just one operation: given a key, it maps the key onto a node. Data location can be easily implemented on top of Chord by associating a key with each data item, and storing the key/data item pair at the node to which the key maps. Chord adapts efficiently as nodes join and leave the system, and can answer queries even if the system is continuously changing. Results from theoretical analysis, simulations, and experiments show that Chord is scalable, with communication cost and the state maintained by each node scaling logarithmically with the number of Chord nodes.}, 94 | booktitle = {Proceedings of the 2001 Conference on Applications, Technologies, Architectures, and Protocols for Computer Communications}, 95 | pages = {149–160}, 96 | numpages = {12}, 97 | location = {San Diego, California, USA}, 98 | series = {SIGCOMM '01} 99 | } 100 | 101 | @inproceedings{bigtable2006, 102 | title = {Bigtable: A Distributed Storage System for Structured Data}, 103 | author = {Fay Chang and Jeffrey Dean and Sanjay Ghemawat and Wilson C. Hsieh and Deborah A. Wallach and Mike Burrows and Tushar Chandra and Andrew Fikes and Robert E. Gruber}, 104 | year = {2006}, 105 | booktitle = {7th {USENIX} Symposium on Operating Systems Design and Implementation (OSDI)}, 106 | pages = {205--218} 107 | } 108 | 109 | @inproceedings{chubby2006, 110 | title = {The Chubby lock service for loosely-coupled distributed systems}, 111 | author = {Mike Burrows}, 112 | year = {2006}, 113 | booktitle = {7th {USENIX} Symposium on Operating Systems Design and Implementation (OSDI)} 114 | } 115 | 116 | @inproceedings{dynamo2007, 117 | author = {DeCandia, Giuseppe and Hastorun, Deniz and Jampani, Madan and Kakulapati, Gunavardhan and Lakshman, Avinash and Pilchin, Alex and Sivasubramanian, Swaminathan and Vosshall, Peter and Vogels, Werner}, 118 | title = {Dynamo: Amazon's Highly Available Key-Value Store}, 119 | year = {2007}, 120 | isbn = {9781595935915}, 121 | publisher = {Association for Computing Machinery}, 122 | address = {New York, NY, USA}, 123 | booktitle = {Proceedings of Twenty-First ACM SIGOPS Symposium on Operating Systems Principles}, 124 | pages = {205–220}, 125 | numpages = {16}, 126 | keywords = {scalability, reliability, performance}, 127 | location = {Stevenson, Washington, USA}, 128 | series = {SOSP '07} 129 | } 130 | 131 | @misc{cassandradocs, 132 | title = {Cassandra Documentation}, 133 | author = {{The Apache Software Foundation}}, 134 | date = {2022}, 135 | url = {https://cassandra.apache.org/doc/4.1/}, 136 | urldate = {2022-08-22}, 137 | } 138 | 139 | @misc{mongodbdocs, 140 | title = {MongoDB Documentation}, 141 | author = {{MongoDB, Inc.}}, 142 | date = {2022}, 143 | url = {https://www.mongodb.com/docs/}, 144 | urldate = {2022-08-23}, 145 | } 146 | 147 | @article{spanner2013, 148 | title = {Spanner: Google’s globally distributed database}, 149 | author = {Corbett, James C and Dean, Jeffrey and Epstein, Michael and Fikes, Andrew and Frost, Christopher and Furman, Jeffrey John and Ghemawat, Sanjay and Gubarev, Andrey and Heiser, Christopher and Hochschild, Peter and others}, 150 | journal = {ACM Transactions on Computer Systems (TOCS)}, 151 | volume = {31}, 152 | number = {3}, 153 | pages = {1--22}, 154 | year = {2013}, 155 | publisher = {ACM New York, NY, USA} 156 | } 157 | 158 | @book{hausenblas2019programming, 159 | title = {Programming Kubernetes: Developing Cloud-Native Applications}, 160 | author = {Hausenblas, Michael and Schimanski, Stefan}, 161 | year = {2019}, 162 | publisher = {O'Reilly Media} 163 | } 164 | 165 | @misc{groupcache, 166 | title = {{groupcache}}, 167 | author = {{Google Inc.}}, 168 | url = {https://github.com/golang/groupcache}, 169 | urldate = {2022-10-11}, 170 | } 171 | 172 | @misc{kerneldocs, 173 | title = {The Linux Kernel documentation}, 174 | author = {{The kernel development community}}, 175 | url = {https://www.kernel.org/doc/html/latest/}, 176 | urldate = {2022-11-06}, 177 | } 178 | 179 | @misc{cockroachdbdocs, 180 | title = {CockroachDB documentation}, 181 | author = {{Cockroach Labs}}, 182 | url = {https://www.cockroachlabs.com/docs}, 183 | urldate = {2022-11-12}, 184 | } 185 | 186 | @article{vogels2008eventually, 187 | title = {Eventually Consistent: Building reliable distributed systems at a worldwide scale demands trade-offs? between consistency and availability.}, 188 | author = {Vogels, Werner}, 189 | journal = {Queue}, 190 | volume = {6}, 191 | number = {6}, 192 | pages = {14--19}, 193 | year = {2008}, 194 | publisher = {ACM New York, NY, USA} 195 | } 196 | 197 | @inproceedings{brewer2000towards, 198 | title = {Towards robust distributed systems}, 199 | author = {Brewer, Eric A}, 200 | booktitle = {PODC}, 201 | volume = {7}, 202 | number = {10.1145}, 203 | pages = {343477--343502}, 204 | year = {2000}, 205 | organization = {Portland, OR} 206 | } 207 | 208 | @inproceedings{ahluwalia2006high, 209 | title = {High availability design patterns}, 210 | author = {Ahluwalia, Kanwardeep Singh and Jain, Atul}, 211 | booktitle = {Proceedings of the 2006 conference on Pattern languages of programs}, 212 | pages = {1--9}, 213 | year = {2006} 214 | } 215 | 216 | @inproceedings{bondi2000characteristics, 217 | title = {Characteristics of scalability and their impact on performance}, 218 | author = {Bondi, Andr{\'e} B}, 219 | booktitle = {Proceedings of the 2nd international workshop on Software and performance}, 220 | pages = {195--203}, 221 | year = {2000} 222 | } 223 | 224 | @article{abadi2009data, 225 | title = {Data management in the cloud: Limitations and opportunities.}, 226 | author = {Abadi, Daniel J}, 227 | journal = {IEEE Data Eng. Bull.}, 228 | volume = {32}, 229 | number = {1}, 230 | pages = {3--12}, 231 | year = {2009}, 232 | publisher = {Citeseer} 233 | } 234 | 235 | @inproceedings{agrawal2004integrating, 236 | title = {Integrating vertical and horizontal partitioning into automated physical database design}, 237 | author = {Agrawal, Sanjay and Narasayya, Vivek and Yang, Beverly}, 238 | booktitle = {Proceedings of the 2004 ACM SIGMOD international conference on Management of data}, 239 | pages = {359--370}, 240 | year = {2004} 241 | } 242 | 243 | @article{jogalekar2000evaluating, 244 | title = {Evaluating the scalability of distributed systems}, 245 | author = {Jogalekar, Prasad and Woodside, Murray}, 246 | journal = {IEEE Transactions on parallel and distributed systems}, 247 | volume = {11}, 248 | number = {6}, 249 | pages = {589--603}, 250 | year = {2000}, 251 | publisher = {IEEE} 252 | } 253 | 254 | @inproceedings{soltesz2007container, 255 | title = {Container-based operating system virtualization: a scalable, high-performance alternative to hypervisors}, 256 | author = {Soltesz, Stephen and P{\"o}tzl, Herbert and Fiuczynski, Marc E and Bavier, Andy and Peterson, Larry}, 257 | booktitle = {Proceedings of the 2Nd ACM SIGOPS/EuroSys european conference on computer systems 2007}, 258 | pages = {275--287}, 259 | year = {2007} 260 | } 261 | 262 | @misc{kubebuilder, 263 | title = {The Kubebuilder Book}, 264 | author = {{The Kubernetes Authors}}, 265 | date = {2022}, 266 | url = {https://book.kubebuilder.io/}, 267 | urldate = {2022-11-23}, 268 | } 269 | 270 | @misc{controllerruntime, 271 | title = {Kubernetes controller-runtime Project}, 272 | author = {{The Kubernetes Authors}}, 273 | date = {2022}, 274 | url = {https://github.com/kubernetes-sigs/controller-runtime}, 275 | urldate = {2022-11-23}, 276 | } 277 | 278 | @misc{kyvernodocs, 279 | title = {Kyverno Documentation}, 280 | author = {{The Kyverno Authors}}, 281 | date = {2022}, 282 | url = {https://kyverno.io/docs/}, 283 | urldate = {2022-11-23}, 284 | } 285 | 286 | @article{schermann2018, 287 | title = {Continuous Experimentation: Challenges, Implementation Techniques, and Current Research}, 288 | author = {Schermann, Gerald and Cito, Jürgen and Leitner, Philipp}, 289 | journal = {IEEE Software}, 290 | year = {2018}, 291 | volume = {35}, 292 | number = {2}, 293 | pages = {26-31}, 294 | } 295 | 296 | @article{adams2015practice, 297 | title = {The practice and future of release engineering: A roundtable with three release engineers}, 298 | author = {Adams, Bram and Bellomo, Stephany and Bird, Christian and Marshall-Keim, Tamara and Khomh, Foutse and Moir, Kim}, 299 | journal = {IEEE Software}, 300 | volume = {32}, 301 | number = {2}, 302 | pages = {42--49}, 303 | year = {2015}, 304 | publisher = {IEEE} 305 | } 306 | -------------------------------------------------------------------------------- /content/30-requirements.md: -------------------------------------------------------------------------------- 1 | # Requirement Analysis {#sec:requirement-analysis} 2 | 3 | Based on the motivation for this study project and the described background, this chapter first describes the current limitations in scaling Kubernetes controllers. 4 | Afterward, it analyzes what is required to make Kubernetes controllers horizontally scalable. 5 | 6 | ## Current Scalability Limitations {#sec:limitations} 7 | 8 | Kubernetes controllers need to prevent uncoordinated and conflicting actions from different instances on the same API objects. 9 | Therefore, controllers currently use leader election mechanisms to determine a single active instance at any given time. 10 | Even if multiple instances are running at the same time, only a single instance carries out the actual work – the current leader. 11 | This means, that a controller's work cannot be distributed across multiple controller instances. 12 | In the context of this thesis, such a setup is referred to as a "singleton controller" setup. 13 | 14 | The most important scalability dimensions of a Kubernetes controller are capacity and throughput. 15 | The capacity of a controller shall be defined as the total number and size of API objects existing in the cluster simultaneously that are watched by the controller. 16 | Increasing a controller's capacity is mainly reflected in increased cache size. 17 | When reaching the upper limit of the controller's capacity, it might run out of memory and stop working entirely. 18 | This sets hard limits for how many API objects can be managed in a single cluster and might limit the ability to support increased usage of the service that the controller is responsible for. 19 | A controller's throughput shall be defined as the rate of reconciliations it performs and watch events it handles. 20 | If API objects are created, updated, and deleted frequently the controller's throughput needs to be high enough to properly handle all reconciliation requests. 21 | When the rate of reconciliation requests exceeds the controller's maximum throughput, the system will become less responsive as reconciliation requests are queued up and delayed. 22 | This results in a decreased user experience and stability of the system. 23 | 24 | Because a controller's work cannot be distributed, its capacity and throughput can only be increased by scaling it vertically, i.e., by adding more resources to the instance. 25 | However, one cannot increase capacity or throughput by adding more controller instances. 26 | I.e., as of now, Kubernetes controllers are not horizontally scalable. 27 | To understand what is required for scaling Kubernetes controllers horizontally, it's important to note which resource dimensions are relevant for scaling. 28 | Generally speaking, the following resource requirements increase with the controller's capacity and throughput: 29 | 30 | +-------------------+------------------------------------------------------------+ 31 | | resource | depends on | 32 | +==================:+============================================================+ 33 | | CPU | rate and size of API objects encoded, decoded | 34 | | | (API requests and watch events), \newline | 35 | | | rate and CPU consumption of actual reconciliations | 36 | +-------------------+------------------------------------------------------------+ 37 | | memory | number and size of API objects stored in the controller's | 38 | | | caches | 39 | +-------------------+------------------------------------------------------------+ 40 | | network bandwidth | rate and size of API requests and watch events | 41 | +-------------------+------------------------------------------------------------+ 42 | 43 | : Resource requirements of a Kubernetes controller {#tbl:scaling-resources} 44 | 45 | As controllers can only be scaled by deploying larger instances, this effectively limits their scalability by the available machine sizes and network bandwidth. 46 | Note, that using bigger machines and broader network connections typically harms cost-efficiency at the top end of the spectrum. 47 | Hence, it is desirable to make controllers horizontally scalable and rather distribute multiple instances across smaller machines instead of deploying bigger instances. 48 | 49 | Additionally, increasing a controller's capacity and throughput also increases the resource footprint of the API server and etcd as outlined in [@tbl:scaling-resources-server]. 50 | However, this study project focuses on the scalability of the controller's side only. 51 | Scalability limitations and implications of the control plane are out of the scope of this thesis. 52 | 53 | +-------------------+-----------------------------------------------------------------------+ 54 | | resource | depends on | 55 | +==================:+=======================================================================+ 56 | | CPU | rate and size of API objects converted, encoded, decoded, \newline | 57 | | | rate and size of watch events dispatched to clients | 58 | +-------------------+-----------------------------------------------------------------------+ 59 | | memory | number and size of API objects stored in the watch cache | 60 | +-------------------+-----------------------------------------------------------------------+ 61 | | network bandwidth | rate and size of API requests and watch events | 62 | +-------------------+-----------------------------------------------------------------------+ 63 | | disk I/O | rate and size of serialized API objects read from and written to disk | 64 | +-------------------+-----------------------------------------------------------------------+ 65 | 66 | : Resource implications on API server and etcd {#tbl:scaling-resources-server} 67 | 68 | ## Requirements for Horizontal Scalability {#sec:requirements} 69 | 70 | To scale Kubernetes controllers horizontally, the restriction of having only a single active instance at any given time needs to be lifted. 71 | For this, a concept for sharding API objects must be introduced which distributes ownership of different objects across multiple instances. 72 | Nevertheless, it must still be prevented that multiple instances own a single API object at the same time or act on it concurrently. 73 | 74 | This section defines precise requirements for a sharding mechanism for Kubernetes controllers. 75 | The requirements themselves are inspired by the required mechanisms for sharding in distributed databases ([@sec:databases]). 76 | If these requirements are met, the resources listed in [@tbl:scaling-resources] can be distributed across multiple instances which makes the controller horizontally scalable. 77 | 78 | \subsection*{\requirement\label{req:membership}Membership and Failure Detection} 79 | 80 | Firstly, the system needs to populate information about the set of controller instances. 81 | In order to distribute object ownership across instances, there must be a mechanism for discovering members of the sharded setup. 82 | Additionally, this mechanism needs to provide information on the availability of individual instances. 83 | Instance failures need to be detected for remediating them and restoring the system's functionality quickly and automatically. 84 | 85 | Because Kubernetes controllers are part of a distributed system and deployed in cloud environments (on commodity hardware), the sharding mechanism must expect that instances are restarted frequently and can fail at any time. 86 | Also, instances can be dynamically added or removed to adapt to growing or decreasing demand. 87 | Furthermore, instances will typically be replaced in quick succession during rolling updates. 88 | For these reasons, voluntary disruptions should be handled specifically and gracefully (scale-down, rolling updates) to achieve fast rebalancing. 89 | 90 | \subsection*{\requirement\label{req:partitioning}Partitioning} 91 | 92 | Secondly, the sharding mechanism must include a partitioning algorithm for determining ownership of a given object based on information about the set of available controller instances. 93 | It must map every sharded API object to exactly one instance. 94 | Additionally, the partitioning algorithm needs to provide a balanced distribution even with a small number of instances (e.g., less than 5). 95 | 96 | As an input for the partitioning algorithm, a partition key is needed. 97 | It needs to apply to all available API resources. 98 | Hence, there should be a generic function for determining the partition key of a given API object. 99 | Furthermore, controllers commonly own and manage objects of different kinds for implementing the logic of a given API resource. 100 | E.g., the `Deployment` controller owns and manages `ReplicaSet` objects. 101 | As controllers watch the owned objects to trigger reconciliation of the owning object on relevant changes to owned objects, the partitioning algorithm must support assigning related objects to the same instance. 102 | For this, all owned objects should map to the same partition key as their owners. 103 | 104 | \subsection*{\requirement\label{req:coordination}Coordination and Object Assignment} 105 | 106 | Next, the sharding mechanism must provide some form of coordination between individual controller instances and assign API objects to the instances based on the partitioning results. 107 | As Kubernetes controllers realize the desired state of the system asynchronously, there is no direct communication of intent between the issuing user and the responsible controller. 108 | Hence, there is no need for partition-based coordination of client requests. 109 | Individual controller instances need to know which objects are assigned to them in order to perform the necessary reconciliations. 110 | However, individual instances don't need to be aware of all object assignments. 111 | The object assignment needs to be transparent and must not change any existing API semantics. 112 | 113 | As described in [@tbl:scaling-resources], the resource requirements of Kubernetes controllers don't only depend on the actual reconciliations but also depend on the controllers' caches and underlying watch connections. 114 | The sharding mechanism can only make controllers horizontally scalable if the instances' caches and watch connections are filtered to only contain and transport the API objects that are assigned to the respective instances. 115 | Otherwise, the mechanism would replicate the cached data and the corresponding resource requirements, which contradicts the main goals of distributing load between instances. 116 | Another important requirement is that individual instances need to be able to retrieve the object assignment information after restarts. 117 | I.e., object assignments must be stored persistently. 118 | 119 | Furthermore, there must not be a single point of failure or bottleneck for reconciliations. 120 | This means, the sharding mechanism must not add additional points of failure on the critical path of API requests and the reconciliations themselves, which would limit the mechanism's scalability again. 121 | During normal operations, reconciliations should not be blocked for a longer period. 122 | 123 | \subsection*{\requirement\label{req:concurrency}Preventing Concurrency} 124 | 125 | Additionally, even when object ownership is distributed across multiple controller instances, concurrent reconciliations for a single object in different instances are still not allowed. 126 | I.e., the purpose of the current leader election mechanism must not be violated. 127 | Only a single instance is allowed to perform mutations on a given object at any given time. 128 | A strictly consistent view on object assignments is not needed for this though. 129 | The only requirement is, that multiple controller instances must not perform writing actions for a single object concurrently. 130 | In other words, the sharding mechanism must not involve the replication of objects. 131 | All API objects are only assigned to a single controller instance. 132 | 133 | \subsection*{\requirement\label{req:scale-out}Incremental Scale-Out} 134 | 135 | The final requirement is that the introduced sharding mechanism provides incremental scale-out properties. 136 | This means that the capacity and throughput of the system increase almost linearly with the added resources, i.e. with every added controller instance. 137 | -------------------------------------------------------------------------------- /content/40-design.md: -------------------------------------------------------------------------------- 1 | # Design {#sec:design} 2 | 3 | This chapter presents a design for implementing sharding in Kubernetes controllers to make them horizontally scalable. 4 | The presented design includes well-known approaches for sharding in distributed databases and applies them to the problem space of Kubernetes controllers. 5 | Approaches are chosen to fulfill the requirements listed in [@sec:requirements]. 6 | First, the overall architecture of the design is introduced. 7 | After that, different aspects of the design are described in detail. 8 | 9 | ## Architecture 10 | 11 | ![Kubernetes controller sharding architecture](../assets/design-overview.pdf) 12 | 13 | The design heavily builds upon existing mechanisms of the Kubernetes API and controllers. 14 | It reuses established approaches from distributed databases and maps them to Kubernetes to implement sharding in a "Kubernetes-native" manner. 15 | This is done to leverage as much of the existing controller infrastructure as possible. 16 | 17 | Firstly, multiple instances of the controller are deployed with the same configuration. 18 | All instances are equal, though one of them is elected to be the "sharder" using the usual lease-based leader election mechanism. 19 | In addition to running the main controllers, the sharder is also responsible for performing sharding-related tasks. 20 | Most importantly, this includes assigning API objects that should be sharded to individual controller instances. 21 | It persists object assignments by setting the `shard` label on the objects themselves to the ID of the responsible instance. 22 | The sharder components themselves are responsible for all API objects and bootstrapping the sharding mechanism. 23 | Hence, the sharder components are only executed as long as the instance is the current leader as they must not run in multiple instances concurrently. 24 | I.e., there is only a single active sharder at any given time and the sharder components in additional instances are on standby. 25 | The sharder uses a metadata-only watch for all the sharded objects to minimize the resulting resource footprint. 26 | 27 | Each controller instance ("shard") maintains an individual lease ("shard lease"), which is used as a heartbeat resource to inform the sharder about the health status of all instances. 28 | The sharder watches the shard leases in addition to the sharded API objects themselves in order to react to state changes, e.g., the addition or removal of shards. 29 | The main controllers in all shards watch and cache only the objects they are responsible for by filtering the watch connection using a label selector on the `shard` label. 30 | 31 | ## Membership and Failure Detection {#sec:des-membership} 32 | 33 | To address requirement \ref{req:membership}, a lease-based membership and failure detection mechanism are used. 34 | The mechanism is heavily inspired by Bigtable but adapted to use Kubernetes `Lease` objects stored via the API server instead of leases stored in Chubby [@bigtable2006]. 35 | 36 | The sharder performs the usual leader election to ensure that there is only one active sharder at any given time. 37 | Additionally, each shard maintains an individual shard lease. 38 | Like usual leader election, the `Lease` API resource with the same fields and semantics are used for this purpose. 39 | However, shard leases use individual names that are equal to the shard's instance ID. 40 | The instance ID needs to be stable between restarts of the shard, so the hostname is used, i.e., the controller's `Pod` name. 41 | On startup, the shards acquire the respective shard lease using the instance ID as name and holder identity. 42 | As with the usual leader election, the shards periodically renew their leases after a configured duration. 43 | As long as the shard actively holds its lease, it watches and reconciles objects that are assigned to itself. 44 | If the shard fails to renew its lease within a configured period, it needs to stop all sharded controllers immediately and terminate. 45 | 46 | The sharder controller watches all shard leases, which allows it to discover all available instances and immediately react to state changes. 47 | As long as a shard holds its lease, the sharder considers it for partitioning. 48 | On shard termination, the shard unsets the lease's holder identity field to release its lease. 49 | This signals a voluntary disruption to the sharder, which immediately removes the instance from the partitioning. 50 | With this, all objects assigned to the terminated shard are moved to other available instances. 51 | 52 | Shard failures are detected by checking the leases' `renewTimestamp` field. 53 | If a shard fails to renew its lease within `leaseDurationSeconds`, the lease is considered expired. 54 | To decrease the sensitivity of the system, the sharder waits for another `leaseDurationSeconds` and allows the instance to renew its lease again. 55 | If the shard comes available again by renewing its lease it is considered ready again. 56 | If the shard fails to renew its lease though, the sharder considers the instance unavailable. 57 | However, before it removes the instance from partitioning, it tries to acquire the shard lease once. 58 | This is done to ensure the API server is functioning and the sharder doesn't act on stale lease data (similar to Bigtable [@bigtable2006]). 59 | If the sharder is able to acquire the shard lease, it considers the instance dead, removes it from partitioning, and moves objects to available instances. 60 | Afterward, the sharder doesn't touch the shard lease anymore. 61 | With this, the shard is allowed to acquire its lease once it comes up again after the lease duration set by the sharder has passed. 62 | If this is successful, the instance is considered ready and included in partitioning again. 63 | 64 | The sharder garbage collects leases of dead shards after a period of inactivity. 65 | With this, leases of voluntarily terminated instances as well as failed instances get cleaned up eventually. 66 | 67 | ## Partitioning 68 | 69 | To address requirement \ref{req:partitioning}, a variant of consistent hashing is used [@karger1997consistent; @stoica2001chord]. 70 | With this, partitioning is done similarly as in Apache Cassandra [@cassandradocs] and Amazon Dynamo [@dynamo2007] but adapted to use Kubernetes API object metadata as input. 71 | Consistent hashing is chosen because it minimizes movement on the addition and removal of instances. 72 | Also, it provides a simple and deterministic algorithm for determining the responsible shard solely based on the set of available instances. 73 | Therefore, no state of the partitioning algorithm must be stored apart from the instance states available through the membership mechanism ([@sec:des-membership]). 74 | The sharder can simply reconstruct the hash ring based on this information after a restart or leader transition without risking inconsistency or unstable assignments. 75 | In order to provide a balanced distribution even with a small number of instances, every instance ID is mapped to a preconfigured number of tokens on the hash ring (virtual nodes) [@stoica2001chord]. 76 | 77 | For determining ownership of a given API object, a partition key is derived from the object's metadata. 78 | By default, it consists of the object's API group, kind, namespace, and name. 79 | This partition key is chosen in order to equally distribute objects of different kinds with the same namespace and name across shards. 80 | Additionally, the object's UID is added to distinguish between different instances of an API object with the same namespace and name. 81 | As all API objects share these common metadata fields, this approach can be applied to all API resources likewise. 82 | Also, it allows the sharder to use lightweight metadata-only watches. 83 | To assign owned objects to the same shard as their owners, the owner's partition key can be determined from the owner reference in the owned object's metadata. 84 | 85 | ## Coordination and Object Assignment 86 | 87 | Realizing coordination and object assignment (requirement \ref{req:coordination}) in Kubernetes controllers is very simple in comparison to distributed databases. 88 | In contrast to sharding in databases, no request coordination is needed because clients don't communicate directly with the controller instances. 89 | Because of this, object assignments don't need to be persisted in a dedicated store or storage section like in Bigtable, MongoDB, or Spanner [@bigtable2006; @mongodbdocs; @spanner2013]. 90 | Furthermore, instances don't need to be aware of assignment information of objects they are not responsible for. 91 | Hence, there is no need to propagate this information throughout the system as it is done in many distributed databases [@dynamo2007; @cassandradocs]. 92 | 93 | In the presented design, object assignment is done by the sharder controller by labeling API objects with the instance ID of the responsible shard. 94 | For each object kind that should be sharded, one sharder controller is started that uses the mechanisms described above for discovering available instances and determining assignments. 95 | Persisting assignments in the API objects themselves are done to make use of existing controller infrastructure for coordination. 96 | By labeling the objects themselves, controllers can simply use a filtered watch to retrieve all API objects that are assigned to them. 97 | For this, a label selector on the `shard` label for the shards' instance ID is added to the controllers' watches. 98 | With this, the controllers' caches and reconciliations are already restricted to the relevant objects and no further coordination between controllers is needed. 99 | Additionally, controllers already rebuild watch connections on failures or after restarts automatically. 100 | This means object assignment information is automatically rebuilt on failures or restarts without any additional implementation. 101 | Using a filter watch and cache additionally addresses requirement \ref{req:scale-out}, as all relevant resource requirements ([@tbl:scaling-resources]) are distributed across multiple instances. 102 | The system's capacity roughly increases linearly with each added instance. 103 | 104 | The sharder itself is not on the critical path for all reconciliations. 105 | As soon as objects are assigned to an instance, the sharder doesn't need to be available for reconciliations to happen successfully. 106 | Reconciliations of new objects might however be delayed by a short period on leader transitions. 107 | As there are multiple instances of the sharder controller on standby, handover generally happens quickly. 108 | Also, handover can be sped up by releasing the leader election lease on voluntary step-down. 109 | 110 | ## Preventing Concurrency {#sec:des-concurrency} 111 | 112 | With the presented design, concurrent writes from different instances to the same API object are already prevented as long as object assignments don't change. 113 | To fulfill requirement \ref{req:concurrency}, concurrent mutating reconciliations must additionally be prevented when moving objects between shards. 114 | In this case, the sharder must ensure that the old instance has stopped working on the given object before another instance picks it up. 115 | 116 | The first case that needs to be considered is when moving objects from a ready shard to another shard. 117 | This can be required for rebalancing during scale-out or a rolling update, i.e. when a new instance is added. 118 | As the controller system is asynchronous and pull-/watch-based, the sharder cannot immediately reassign objects. 119 | The watch connection and cache of the old instance might receive the reassignment event later than the new instance, which could lead to concurrent and conflicting reconciliations. 120 | In other words, the sharder can't know if and when an instance has stopped working on an object that is supposed to be moved. 121 | To address this challenge, the old instance needs to acknowledge the reassignment and confirm it has stopped working on the object. 122 | When the sharder needs to move an object from a ready shard to another, it adds a `drain` label indicating that the object should be moved. 123 | As soon as the old shard observes the `drain` label it must remove it from the object as well as the `shard` label, marking the object as unassigned. 124 | When the sharder observes that the object is unassigned it reassigns it to the desired instance as it does for new objects. 125 | Following this protocol, concurrent reconciliations by the old and the new shard are prevented. 126 | Also, it facilitates fast rebalancing with ready and responsive instances. 127 | 128 | A second case to consider is when moving objects from a terminating instance to a ready instance. 129 | This can be required for rebalancing during scale-in or a rolling update, i.e. when an existing instance is removed. 130 | In this case, the shard releases its shard lease to signal voluntary termination to the sharder. 131 | As soon as the sharder observes this change to the shard lease it immediately reassigns API objects to another instance. 132 | A `drain` operation is not performed in this case because the instance has already acknowledged that it stopped working on the objects by releasing its shard lease. 133 | 134 | The last case that needs to be considered is when moving objects from a dead instance, i.e. for rebalancing after an instance failure. 135 | Once the sharder has detected a shard failure and acquired the shard lease as described in [@sec:des-membership], objects are reassigned immediately without performing a `drain` operation. 136 | If the shard eventually becomes ready again, objects can be assigned to it again once it acquires its lease just like when a new instance is added. 137 | A special case occurs when an instance failure is detected after initiating a `drain` operation. 138 | In this case, the sharder removes the `drain` label by itself when updating the `shard` label to prevent performing the `drain` operation with the shard that the object is supposed to be assigned to. 139 | -------------------------------------------------------------------------------- /pandoc/ieee.csl: -------------------------------------------------------------------------------- 1 | 2 | 458 | -------------------------------------------------------------------------------- /content/60-evaluation.md: -------------------------------------------------------------------------------- 1 | # Evaluation {#sec:evaluation} 2 | 3 | This chapter describes how the presented implementation ([chapter @sec:implementation]) is used for the evaluation of the proposed design ([chapter @sec:design]). 4 | First, the deployment environment is described. 5 | Next, the monitoring setup is introduced which is used to observe the webhosting operator and to take measurements. 6 | After that, the conducted experiment is described in detail. 7 | Last, the experiment's results are presented and discussed. 8 | 9 | ## Deployment {#sec:deployment} 10 | 11 | For evaluation of the proposed design, the webhosting operator implementation is deployed to a Kubernetes cluster along with other tools for monitoring and experimentation. 12 | The Kubernetes cluster[^shoot] is managed by Gardener[^gardener] and runs on Google Cloud Platform (region `europe-west1`). 13 | The cluster runs Kubernetes version `1.24`. 14 | 15 | For isolation purposes and accurate measurements during load tests, the following worker pools are used to run the different components: 16 | 17 | | worker pool | instance type | components | count | 18 | |-------------|---------------|-------------------------------------|-------| 19 | | system | n1-standard-8 | cluster system, monitoring, ingress | 1 | 20 | | operator | n1-standard-4 | webhosting operator | 1 | 21 | | websites | n1-standard-8 | website deployments | 10 | 22 | 23 | : Worker pools of the evaluation cluster {#tbl:worker-pools} 24 | 25 | Both the `operator` and `websites` worker pools are configured with taints [@k8sdocs] to repel all workload pods by default. 26 | With this, all pods are scheduled to the system worker pool if not configured otherwise. 27 | Kyverno policies [@kyvernodocs] are used to add tolerations and scheduling constraints to all webhosting operator and website pods so that they are scheduled to the respective dedicated worker pools. 28 | To reduce the resource requirements of load tests, the websites worker pool is configured to run up to 800 pods per node. 29 | Additional policies are used to replace the `nginx` image in website pods with the `pause` image during load tests. 30 | With this, websites are not functional but have a low memory footprint without affecting the experiment's measurements. 31 | 32 | [^shoot]: [Cluster specification](https://github.com/timebertt/kubernetes-controller-sharding/blob/v1.0/webhosting-operator/shoot.yaml) 33 | [^gardener]: [https://github.com/gardener/gardener](https://github.com/gardener/gardener) 34 | 35 | ## Monitoring 36 | 37 | For observing and measuring the implementation, a monitoring setup is deployed in the Kubernetes cluster in addition to the webhosting operator itself. 38 | The monitoring setup serves two purposes: for general evaluation that the sharded controllers are working as desired and for providing accurate measurements of the operator's resource usage. 39 | 40 | The deployed monitoring setup is based on kube-prometheus[^kube-prometheus], which is a collection of monitoring tools and configurations for Kubernetes clusters. 41 | Most importantly, it includes a Prometheus [@promdocs] instance which collects metrics for containers running on the cluster and metrics about the state of Kubernetes API objects. 42 | Container metrics are scraped from the kubelet's cadvisor endpoint [@k8sdocs], kube-state-metrics[^kube-state-metrics] is used as an exporter for metrics about API objects. 43 | Furthermore, the setup includes a Grafana [@grafanadocs] instance, which visualizes the metrics collected by prometheus in different dashboards. 44 | 45 | ![Webhosting dashboard](../assets/dashboard-webhosting.png){#fig:dashboard-webhosting} 46 | 47 | In addition to the components included in kube-prometheus, the webhosting exporter[^webhosting-exporter] is deployed which exposes metrics about the state of websites. 48 | It is implemented using the kube-state-metrics library and complements the metrics offered by kube-state-metrics by adding similar metrics about the webhosting operator's API objects. 49 | For example, one metric indicates the websites' phase, and another one states which shard websites are assigned to. 50 | Additionally, the webhosting exporter is used to collect metrics about the state of the operator's shards, which is determined from the individual shard leases. 51 | 52 | ![Sharding dashboard](../assets/dashboard-sharding.png){#fig:dashboard-sharding} 53 | 54 | Finally, Prometheus is configured to collect metrics from the webhosting operator as well. 55 | It exposes metrics from the controller-runtime library about controllers, work queues, and the Kubernetes client ([@fig:dashboard-controller-runtime]). 56 | For example, these metrics offer information on the rate and duration of reconciliations, and for how long objects are queued before being reconciled. 57 | Additionally, metrics are added about the sharder controllers' actions, i.e. assignment and drain operations. 58 | 59 | ![Controller-runtime dashboard](../assets/dashboard-controller-runtime.png){#fig:dashboard-controller-runtime} 60 | 61 | For general evaluation of the implementation, a sample generator[^samples-generator] is implemented to create a configurable number of random website objects in different namespaces. 62 | [@Fig:dashboard-webhosting] shows the overview dashboard over webhosting API objects and their state based on metrics exposed by webhosting exporter and kube-state-metrics. 63 | It demonstrates, that all websites are successfully reconciled by the sharded operator and reach the `Ready` phase as expected. 64 | The sharding overview in [@fig:dashboard-sharding] indicates that all 3 shards are healthy and properly maintain their shard lease. 65 | It also shows that responsibility for website objects is well-distributed across all 3 shards, with each shard being assigned roughly a third of all objects. 66 | 67 | ```yaml 68 | queries: 69 | - name: cpu 70 | query: sum(rate(container_cpu_usage_seconds_total{namespace="webhosting-system", container="manager"}[2m])) by (pod) 71 | - name: memory 72 | query: sum(container_memory_rss{namespace="webhosting-system", container="manager"}) by (pod) 73 | - name: network_receive 74 | query: sum(irate(container_network_receive_bytes_total{namespace="webhosting-system", pod=~"webhosting-operator-.+"}[2m])) by (pod) 75 | - name: network_transmit 76 | query: sum(irate(container_network_transmit_bytes_total{namespace="webhosting-system", pod=~"webhosting-operator-.+"}[2m])) by (pod) 77 | ``` 78 | 79 | : Queries configuration for experiments {#lst:measure-queries} 80 | 81 | Finally, the measure[^measure] tool is implemented for retrieving the relevant measurements from the Prometheus metrics store via the HTTP API [@promdocs]. 82 | It fetches time series over a given time range and stores the result matrices in CSV-formatted files for further analysis and visualization. 83 | [@Lst:measure-queries] shows the query configuration that is used to determine the operator's resource usage in the following experiment. 84 | 85 | [^kube-prometheus]: [https://github.com/prometheus-operator/kube-prometheus](https://github.com/prometheus-operator/kube-prometheus) 86 | [^kube-state-metrics]: [https://github.com/kubernetes/kube-state-metrics](https://github.com/kubernetes/kube-state-metrics) 87 | [^webhosting-exporter]: [webhosting exporter source code](https://github.com/timebertt/kubernetes-controller-sharding/tree/v1.0/webhosting-operator/cmd/webhosting-exporter) 88 | [^samples-generator]: [samples generator source code](https://github.com/timebertt/kubernetes-controller-sharding/tree/v1.0/webhosting-operator/cmd/samples-generator) 89 | [^measure]: [measure source code](https://github.com/timebertt/kubernetes-controller-sharding/tree/v1.0/webhosting-operator/cmd/measure) 90 | 91 | ## Experiment 92 | 93 | Using the described deployment and monitoring setup, an experiment is conducted to evaluate the sharding implementation in the webhosting operator. 94 | The experiment aims to determine whether the relevant resources from [@tbl:scaling-resources] are well-distributed across the individual instances. 95 | In other words, the experiment verifies if the system's capacity and throughput are increased with the resources used by the additional instances. 96 | It assesses how well the sharded operator scales horizontally. 97 | 98 | For this purpose, the experiment tool[^experiment] is implemented to conduct load tests for the webhosting operator. 99 | The tool generates load by constantly creating objects and triggering reconciliations over a certain time span. 100 | It thereby increases the capacity and throughput of the system to measure the resulting resource usage. 101 | As described in [@sec:deployment], the webhosting operator is deployed on nodes isolated from all other workloads. 102 | This ensures that the operator is able to consume as many resources as it needs to handle the actual capacity and throughput of the system. 103 | By this, the experiment allows concluding how much resources are needed to handle a certain capacity and throughput, and how both increase with the number of resources added to the system. 104 | 105 | For now, only one experiment scenario is implemented, which is called `base`[^base-scenario]. 106 | In this scenario, the tool generates 50 random `Theme` objects and 20 random project namespaces. 107 | Over 10 minutes it then generates random `Website` objects in different project namespaces using different `Themes`. 108 | To make the load tests more realistic, `Websites` are also randomly deleted, though at a lower rate than creation. 109 | During the load test, the tool generates 14 random `Websites` per second and deletes 1 object per second, hence there are about 7800 `Websites` after 10 minutes. 110 | To simulate frequent changes to `Website` objects by users, the tool triggers individual reconciliations of each object once per minute but 100 reconciliations per second at maximum. 111 | Additionally, the tool also simulates occasional changes to `Theme` objects by service administrators. 112 | For this, it picks one random `Theme` per minute and mutates its specification. 113 | As described in [@sec:webhosting-operator], this update triggers reconciliations for all `Website` objects referencing the `Theme`. 114 | After 10 minutes, the tool generates about 103 reconciliations per second on average with bursts of roughly 256 reconciliations per second. 115 | This load is sustained for 5 more minutes before all objects generated by the experiment are cleaned up. 116 | [@Fig:base-websites] shows how the amount of websites is increased over the experiment's timespan and how they are distributed across the operator's shards. 117 | 118 | [^experiment]: [experiment source code](https://github.com/timebertt/kubernetes-controller-sharding/tree/v1.0/webhosting-operator/cmd/experiment) 119 | [^base-scenario]: [base scenario source code](https://github.com/timebertt/kubernetes-controller-sharding/tree/v1.0/webhosting-operator/pkg/experiment/scenario/base) 120 | 121 | ![Load generated by base scenario](../results/base-websites.pdf){#fig:base-websites} 122 | 123 | [@Tbl:metrics] shows which metrics are used for measurements of the operator's resource usage during the experiment. 124 | All the metrics originate from kubelet's cadvisor endpoint. 125 | For memory measurements, the resident set size (RSS) metric is used instead of the working set size (WSS) [@kerneldocs]. 126 | This is because the WSS includes pages that are already freed by the process but not reclaimed by the kernel yet. 127 | The RSS on the other hand does not include such pages and hence provides a more precise measurement of how much memory the operator process actually requires. 128 | 129 | | resource | metric | 130 | |-------------------------------|----------------------------------------| 131 | | CPU | container_cpu_usage_seconds_total | 132 | | memory | container_memory_rss | 133 | | network bandwidth received | container_network_receive_bytes_total | 134 | | network bandwidth transmitted | container_network_transmit_bytes_total | 135 | 136 | : Experiment resource usage measurements {#tbl:metrics} 137 | 138 | Network bandwidth measurements are split by direction: one measurement for received bytes and one for transmitted bytes. 139 | These metrics include network transfer related to Prometheus scraping the pods' metrics endpoints. 140 | However, this can be neglected in this experiment as the transfer only occurs every 10 seconds and includes a short plain text in comparison to the size of the transferred API objects. 141 | Note that these resource measurements only cover the operator processes but not the Kubernetes API server. 142 | Investigating the resource implications of sharding on the control plane components is out of the scope of this thesis. 143 | 144 | The load test is executed twice: once for the singleton operator and once for the sharded operator. 145 | For this, the deployment mechanism offers a switch for deploying the operator in either mode. 146 | In the sharded mode, the operator is deployed with three instances, and in the singleton mode only with one instance. 147 | Note that singleton operators are typically deployed with multiple instances for fast failovers (active-passive HA setup) in production environments. 148 | As the passive instances of the operator also have a given base resource usage, the total resource usage of the system would be higher than in the conducted experiments. 149 | After each load test, the measurements are retrieved from Prometheus via the measure tool. 150 | Then, the measured resource usage of both tests is compared. 151 | 152 | ## Results 153 | 154 | This section compares the resource usage by individual pods in both tests for each measured resource ([@tbl:metrics]). 155 | 156 | ![CPU usage by pod](../results/base-cpu.pdf){#fig:base-cpu} 157 | 158 | First, [@fig:base-cpu] compares the CPU usage of pods in both setups. 159 | In the sharded setup, the pod with the highest CPU usage is the active sharder. 160 | The comparison shows that the sharder's CPU usage is almost as high as the singleton's usage. 161 | It only consumed 12% less CPU time than the singleton over the entire test duration. 162 | The other two shards' total CPU time is about 48% of the singleton's usage. 163 | 164 | ![Memory usage by pod](../results/base-memory.pdf){#fig:base-memory} 165 | 166 | The comparison of the pods' memory usage ([@fig:base-memory]) reveals that the sharder's memory usage is almost as high as the singleton's memory usage – similar to the CPU usage. 167 | On average, the sharder's RSS is about 89% of the singleton's RSS. 168 | The other two shards use significantly less memory: on average, roughly 40% of the singleton. 169 | 170 | ![Network bandwidth by pod](../results/base-network.pdf){#fig:base-network} 171 | 172 | Finally, [@fig:base-network] compares the used network bandwidth of both setups by direction. 173 | It shows that the amount of transmitted bytes is very similar in each instance of the sharded setup, while the sharder receives significantly more bytes than the other two shards. 174 | The total bytes received by the sharder are 89% of the singleton's total, while the other two shards receive 42% as many bytes as the singleton. 175 | The sharded operator instances transmit about 44% as many bytes as the singleton. 176 | 177 | ## Discussion 178 | 179 | The results presented in the previous section demonstrate that the proposed design for sharding in Kubernetes controllers achieves a distribution of the required resources across controller instances. 180 | First of all, the implemented mechanisms realize a good distribution of responsibility for API objects among the three shards. 181 | It is shown that the individual shards of the system require roughly half as many resources in comparison to the singleton controller. 182 | With that, the presented sharding mechanism overcomes the current scalability limitation of Kubernetes controllers and makes them horizontally scalable. 183 | Using such an architecture allows increasing the system's capacity and throughput by adding more controller instances. 184 | 185 | However, the results also highlight that the instance that is the active sharder consumes almost as many resources as the singleton controller. 186 | This limits the scalability of the system again by the available machine sizes and network bandwidth similar to the singleton controller architecture. 187 | Hence, the scalability limitations described in [@sec:limitations] are not removed entirely. 188 | It is important to note, that in the evaluated deployment setup, the sharder runs both the sharder and shard components. 189 | I.e., it is responsible for assigning objects to individual instances as well as running the actual object controllers just like the other instances. 190 | This results in significantly higher resource consumption in comparison to the other instances that only run the shard components. 191 | 192 | Nevertheless, the presented design does not prescribe that the sharder must also act as a shard at the same time. 193 | The deployment setup could be adapted to run the shard and sharder in dedicated instance sets (i.e., `Deployments`). 194 | Based on the presented experiment results, it is expected that this results in better distribution of resource requirements. 195 | With that, the original scalability limitation in the vertical direction can be overcome entirely and the system's scalability can be increased even further. 196 | 197 | The results also show that the two pods that only run the shard components have very similar resource usage. 198 | In a typical singleton controller setup with multiple instances for active-passive HA, there is a much more heterogeneous resource usage as the passive instances consume almost no resources in comparison to the leader. 199 | With this, the sharded setup also brings advantages for vertical scaling as equally-sized instances can be right-sized more easily to minimize wasting reserved resources. 200 | 201 | Last, it is worth noting that the simulated load is still very low in comparison to some production deployments of controllers. 202 | With even more and larger API objects in the system, the difference between the sharder's and singleton's resource usage might be more significant. 203 | -------------------------------------------------------------------------------- /content/50-implementation.md: -------------------------------------------------------------------------------- 1 | # Implementation 2 | 3 | This chapter describes how the design presented in [chapter @sec:design] is implemented in practice. 4 | First, the webhosting operator is introduced which serves as an example operator for implementing sharding in Kubernetes controllers. 5 | Then, the overall architecture of the implementation is described, followed by detailed descriptions of the most important aspects. 6 | The presented implementation is used for evaluation in [chapter @sec:evaluation]. 7 | 8 | ## Webhosting Operator {#sec:webhosting-operator} 9 | 10 | [^webhosting-operator]: [https://github.com/timebertt/kubernetes-controller-sharding/tree/v1.0](https://github.com/timebertt/kubernetes-controller-sharding/tree/v1.0) 11 | 12 | To implement and evaluate the presented design, an operator is needed that serves as an example. 13 | The sample operator should fulfill the following criteria so that the most important aspects of sharding in Kubernetes controllers can be covered and evaluated: 14 | 15 | 1. The operator should be composed of a single controller performing reconciliations for one custom resource. 16 | With this, the operator's capacity and throughput can be measured and compared more easily as they are mainly determined by the number of objects and event rate for this resource. 17 | 18 | 2. In addition to watching the reconciled objects, the controller should also watch all other relevant objects. 19 | This includes owned objects, that the controller manages for realizing the intent of the reconciled objects. 20 | Also, it should watch objects that might be referenced by multiple reconciled objects. 21 | Both of these are common patterns in controllers, which makes the example operator a good representative. 22 | 23 | 3. Finally, the operator needs to follow controller best practices [@k8sdesign]. 24 | Most importantly, it should be level-based, meaning it performs reconciliations purely based on the desired and currently observed state but independent of observed changes to these states. 25 | However, it should be edge-triggered for performance and responsiveness, i.e. start reconciliations based on relevant change events. 26 | Reconciliations should be short and must not wait for the desired and actual state to converge. 27 | Lastly, the operator should hold all relevant state in memory, meaning it should cache all API resources it works with. 28 | Following these best practices in the example operator is important because many controllers are implemented in compliance with them. 29 | Hence, the evaluation can only provide meaningful insights if the measured implementation follows the best practices. 30 | 31 | Based on these criteria, the webhosting operator[^webhosting-operator] was designed and built. 32 | The operator and its resources are modeled to form a webhosting-like platform, where customers can create and manage websites in a declarative manner via the Kubernetes API. 33 | Websites reside in a project namespace, have a name, and specify a website theme, which defines color and font family. 34 | The desired state of websites is declared via Kubernetes resources and the operator manages the required web servers and exposes them to the internet. 35 | 36 | ```yaml 37 | apiVersion: webhosting.timebertt.dev/v1alpha1 38 | kind: Theme 39 | metadata: 40 | name: exciting 41 | spec: 42 | color: darkcyan 43 | fontFamily: Menlo 44 | --- 45 | apiVersion: webhosting.timebertt.dev/v1alpha1 46 | kind: Website 47 | metadata: 48 | name: homepage 49 | namespace: project-foo 50 | spec: 51 | theme: exciting 52 | ``` 53 | 54 | : Example webhosting operator objects {#lst:webhosting} 55 | 56 | Three API resources are involved when managing websites using the webhosting operator: 57 | 58 | 1. `Namespaces` are used to separate the websites of different customers and projects. 59 | I.e., one could provide new customers with access to a dedicated namespace upon registration to the service. 60 | 61 | 2. `Theme` is a custom resource of the webhosting operator and part of the `webhosting.timebertt.dev` API group. 62 | `Themes` are cluster-scoped and can be referenced by `Websites` in all namespaces. 63 | They are managed by the service administrator and configure an HTML color and font family. 64 | 65 | 3. `Websites` are also part of the operator's custom API group, but are namespaced. 66 | A `Website` object represents a single website managed by a customer in a respective project namespace. 67 | It references exactly one of the offered `Themes`. 68 | For each `Website` object, the operator creates and configures an nginx `Deployment` to host a simple static website that displays the website's name and technical ID ("server name"). 69 | The website uses the color of the referenced `Theme` as a background color as well as the declared font family. 70 | Additionally, the `Deployment` is exposed via an `Ingress` object on the URL path `//`. 71 | 72 | ![Sample website managed by webhosting operator](../assets/sample-website.png) 73 | 74 | The webhosting operator contains a single controller for realizing the webhosting functionality. 75 | On each reconciliation of a `Website` object, it ensures that owned objects of the `Website` are correctly configured: the nginx `ConfigMap` and `Deployment` as well as `Service` and `Ingress` for exposing the `Deployment`. 76 | It adds owner references from all of these objects to the owning `Website` object. 77 | In addition to watching `Website` objects, it also watches these owned objects and enqueues the owning `Website` for reconciliation on relevant changes, e.g. when the `Deployment` object's readiness status changes. 78 | This facilitates self-healing capabilities of websites, as the operator ensures that all required objects have the desired state. 79 | Additionally, it allows the controller to immediately report the status of the `Website` object in the `status.phase` field based on the status of the nginx `Deployment`. 80 | 81 | Furthermore, the controller watches all available `Themes` and enqueues all `Website` objects that reference a `Theme` for reconciliation, if its specification changes. 82 | With this, changes to `Themes` are immediately rolled out to all `Websites` that use them. 83 | Additionally, `Website` reconciliations can be triggered by annotation changes for load test purposes. 84 | The reconciliations themselves are very short, making the controller responsive and scalable. 85 | 86 | ## Architecture 87 | 88 | The webhosting operator is based on the controller-runtime library. 89 | The library provides reusable constructs for building custom Kubernetes controllers and operators in the go programming language. 90 | In controller-runtime, the `Manager` is the most important construct. 91 | A `Manager` is initialized and started once per operator as the root component and manages all other components like controllers, webhook server and handlers, leader election, metrics endpoints, loggers, etc. [@kubebuilder; @controllerruntime] 92 | 93 | Individual controllers can be implemented by creating a corresponding `Controller` construct and configuring a `Reconciler`, watches, and event handlers. 94 | The `Reconciler` contains the business logic of a controller. 95 | It is invoked with a reconciliation request containing an object's name and namespace. 96 | On each invocation, it has to compare the actual state of the system with the desired state of the given object and perform actions to converge both, i.e. perform reconciliation of the object. 97 | The configured watches and event handlers are responsible for enqueuing reconciliation requests in response to watch events. 98 | Typically, `Predicates` are used to filter for relevant watch events emitted by the API server. 99 | The `Controller` itself ensures that all components including cache, event handlers, work queue, and worker routines are correctly set up. 100 | All `Controllers` are registered with the `Manager` which then ensures that the controllers are started when the leader election is won and stopped as soon as the leader election is lost. 101 | It also injects shared dependencies like client, cache and loggers into the individual controllers. [@controllerruntime] 102 | 103 | As part of this thesis, the presented design ([chapter @sec:design]) was fully implemented generically in the controller-runtime library that allows reusing the mechanisms in all operators based on the library. 104 | The webhosting operator makes use of the implemented sharding mechanisms in controller-runtime for demonstration and evaluation purposes. 105 | 106 | For adding controller sharding to an operator based on controller-runtime, there are two places involved: when configuring the `Manager` and when setting up a sharded `Controller`. 107 | First, sharding has to be enabled by setting `manager.Options.Sharded=true`. 108 | With this, the `Manager` is configured to maintain a shard lease while it is running ([@sec:impl-shard-lease]). 109 | Furthermore, it adds a cache to the `Manager` which uses a label selector `shard=` for all objects. 110 | This cache is then used by sharded controllers as a filtered cache that only contains objects that the shard is responsible for, hence it is referred to as the sharded cache. 111 | In controller-runtime, the cache is empty on startup and watches are started on-demand when a controller reads from the cache. 112 | The `Manager`'s default cache (unfiltered) is still available for retrieval of non-sharded objects. 113 | However, as long as controllers use the correct cache when reading objects, watches will only be started in one cache for any `GroupVersionKind` and thus are filtered correctly. 114 | For development purposes, the shard ID can optionally be overwritten. 115 | Also, the `ShardMode` option allows configuring whether the manager should only run the sharder components (under leader election), only the shard components, or both (default). 116 | 117 | ```go 118 | mgr, err := manager.New(restConfig, manager.Options{ 119 | // enable sharding for this manager 120 | Sharded: true, 121 | // optionally configure overwrites for development purposes 122 | ShardID: "custom-shard-id", 123 | ShardMode: sharding.ModeSharder, 124 | }) 125 | ``` 126 | 127 | : Configuring the controller-runtime manager for sharding {#lst:manager-setup} 128 | 129 | There are two sharder components: the lease controller and the sharder controllers. 130 | Both are added in all instances, but they are only running when the instance has acquired leadership. 131 | The lease controller ([@sec:impl-lease-controller]) is running only once per leader and is mainly responsible for acquiring leases of unavailable shards ([@sec:des-membership]). 132 | The sharder controllers are coupled to the setup of the sharded controllers themselves, though they only run in the leader. 133 | 134 | ```go 135 | controller, err := builder.ControllerManagedBy(mgr). 136 | For(&webhostingv1alpha1.Website{}, builder.Sharded{}). 137 | // watch deployments in order to update phase on relevant changes 138 | Owns(&appsv1.Deployment{}, builder.Sharded{}). 139 | // ... 140 | // watch themes to roll out theme changes to all referencing websites 141 | Watches( 142 | &source.Kind{Type: &webhostingv1alpha1.Theme{}}, 143 | handler.EnqueueRequestsFromMapFunc(r.MapThemeToWebsites), 144 | builder.WithPredicates(predicate.GenerationChangedPredicate{}), 145 | ). 146 | Build(reconciler) 147 | ``` 148 | 149 | : Setup of a sharded controller {#lst:controller-setup} 150 | 151 | For setting up a sharded controller, the `builder` package must be used. 152 | With this, sharded controllers are configured like normal controllers however with an additional `builder.Sharded` option. 153 | This option is supplied to the controller's main object type (in `For`) and all additional object types that should be sharded (in `Owns` and `Watches`). 154 | When setting the `Sharded` option, the builder performs the following tasks: 155 | 156 | - it adds a sharder controller ([@sec:impl-sharder-controllers]) for the corresponding `GroupVersionKind` 157 | - it modifies the main object controller ([@sec:impl-object-controller]) to properly handle the sharding mechanisms 158 | 159 | The next sections explain the implementation of all components in detail. 160 | 161 | ## Shard Lease {#sec:impl-shard-lease} 162 | 163 | When configured for sharding, the manager keeps performing leader election ([@sec:leader-election]) and runs the sharder components only when it has successfully acquired leadership. 164 | In addition to that, the manager also maintains an individual shard lease ([@sec:des-membership]). 165 | The mechanism executes the same leader election code, however with an individual lease name. 166 | For shard leases, the instance's hostname is used for both the lease name and holder identity. 167 | As long as the manager is able to renew the shard lease it keeps running the sharded controllers. 168 | 169 | When adding controllers and other `Runnables` to a manager via `manager.Add` they can signal whether they need to run under leader election or not by implementing the `LeaderElectionRunnable` interface. 170 | For example, controllers implement this interface by default to always run with leader election. 171 | A new interface is introduced to distinguish between usual and sharded controllers: `ShardedRunnable`. 172 | 173 | ```go 174 | // ShardedRunnable knows if a Runnable needs to be run in the sharded mode. 175 | type ShardedRunnable interface { 176 | // IsSharded returns true if the Runnable needs to be run in the sharded mode. 177 | IsSharded() bool 178 | } 179 | ``` 180 | 181 | : The ShardedRunnable interface {#lst:sharded-runnable} 182 | 183 | When creating a sharded controller using the builder, the manager recognizes the `SharedRunnable` implementation and starts it as soon as the shard lease has been acquired even if the instance is currently not the leader. 184 | Similar to the usual leader election, the manager process exits immediately when it fails to renew its shard lease. 185 | When the manager is stopped, it engages a graceful termination procedure and releases its shard lease to signal voluntary disruption, i.e. it sets `spec.holderIdentity=null`. 186 | 187 | ## Lease Controller {#sec:impl-lease-controller} 188 | 189 | The lease controller watches the shard leases that individual instances maintain. 190 | For a given shard lease, it first determines the instance's state and then adds the state to the lease object as the `state` label for observability purposes. 191 | The shard states are defined by the following rules: 192 | 193 | - `Ready`: the lease is held by the shard itself (`metadata.name` is equal to the `holderIdentity`) and has not expired (`renewTime + leaseDurationSeconds` has not passed yet) 194 | - `Expired`: the lease is held by the shard but has expired up to `leaseDurationSeconds` ago 195 | - `Uncertain`: the lease is held by the shard and has expired more than `leaseDurationSeconds` ago 196 | - `Dead`: the lease is not held by the shard anymore, either because the shard released its lease or it was acquired by the sharder (lease controller) 197 | - `Orphaned`: the lease is in state `Dead` and has expired at least 1 minute ago 198 | 199 | The main responsibility of the lease controller is to act on leases of unavailable shards. 200 | Once shard leases transition to state `Uncertain`, it acquires the lease by setting `holderIdentity` to `sharder`. 201 | For this, it uses a `leaseDuration` twice as long as the `leaseDuration` of the shard. 202 | Acquiring the shard lease is done to ensure the API server is functioning before removing the shard from partitioning ([@sec:des-membership]). 203 | Additionally, the lease controller deletes shard leases in the state `Orphaned` to not pollute the system. 204 | The shard controller reconciles leases as soon as their state changes, either because of a create/update event or because one of the relevant durations has passed. 205 | 206 | ## Sharder Controllers {#sec:impl-sharder-controllers} 207 | 208 | For every sharded object kind, the controller builder adds a sharder controller that runs under leader election. 209 | It is responsible for assigning the sharded objects to available shards by setting the `shard` label based on the shard state information. 210 | For this, it starts a lightweight metadata-only watch/cache for the respective object kind. 211 | As soon as objects are created or need to be re-assigned the controller performs reconciliation. 212 | Additionally, it watches the shard leases and enqueues relevant sharded objects when a shard's availability changes. 213 | 214 | On every reconciliation, the controller lists all shard leases from the cache. 215 | It then constructs a hash ring including all shards that are not in the state `Dead` or `Orphaned`. 216 | As this operation is executed for every sharder reconciliation, it is saved for reuse in a dedicated cache that is shared across all sharder controllers belonging to a given sharded controller. 217 | The cached ring is recalculated when a lease is updated. 218 | In the hash ring, 100 tokens are added per shard by default for better distribution of objects across a low number of shards (inspired by groupcache [@groupcache]). 219 | 220 | ```go 221 | type Hash func(data []byte) uint64 222 | 223 | type Ring struct { 224 | hash Hash // defaults to 64-bit implementation of xxHash (XXH64) 225 | tokensPerNode int // defaults to 100 226 | 227 | tokens []uint64 228 | tokenToNode map[uint64]string 229 | } 230 | 231 | func (r *Ring) AddNodes(nodes ...string) { 232 | for _, node := range nodes { 233 | for i := 0; i < r.tokensPerNode; i++ { 234 | t := r.hash([]byte(fmt.Sprintf("%s-%d", node, i))) 235 | r.tokens = append(r.tokens, t) 236 | r.tokenToNode[t] = node 237 | } 238 | } 239 | 240 | // sort all tokens on the ring for binary searches 241 | sort.Slice(r.tokens, func(i, j int) bool { 242 | return r.tokens[i] < r.tokens[j] 243 | }) 244 | } 245 | ``` 246 | 247 | : Adding shards to the hash ring {#lst:ring-add} 248 | 249 | After calculating the hash ring, the object is mapped to a partition key. 250 | For the main object kind of sharded controllers, the partition key follows this pattern: 251 | 252 | `.///` 253 | 254 | For any owned objects of the main object kind, the partition key of the owner is calculated using the object's owner references. 255 | The calculated partition key is then used to determine the desired shard for the object by hashing the key onto the hash ring. 256 | 257 | ```go 258 | func (r *Ring) Hash(key string) string { 259 | // Hash key and walk the ring until we find the next virtual node 260 | h := r.hash([]byte(key)) 261 | 262 | // binary search 263 | i := sort.Search(len(r.tokens), func(i int) bool { 264 | return r.tokens[i] >= h 265 | }) 266 | 267 | // walked the whole ring 268 | if i == len(r.tokens) { 269 | i = 0 270 | } 271 | 272 | return r.tokenToNode[r.tokens[i]] 273 | } 274 | ``` 275 | 276 | : Consistent hashing of object keys {#lst:ring-hash} 277 | 278 | If the object is not assigned yet, the sharder controller simply patches the object's `shard` label to the desired shard. 279 | However, if the object is already assigned to a different shard, the sharder controller first adds the `drain` label to the object to wait for acknowledgment by the current shard ([@sec:des-concurrency]). 280 | This operation is only performed for the main object kind of the sharded controller, as it does not perform mutating actions on owned objects as long as it isn't responsible for the owning object. 281 | Hence, concurrency is already prevented by the drain operation of the main object and doesn't need to be handled separately for owned objects. 282 | 283 | ## Object Controller {#sec:impl-object-controller} 284 | 285 | The actual sharded controller is modified by the builder to handle the sharding mechanisms. 286 | First, the controller implements the `ShardedRunnable` interface so that all instances run the controller and not only the leader. 287 | Also, the builder constructs a cache that automatically delegates all read operations to either the sharded (filtered) cache or unfiltered cache according to whether they are configured to be sharded or not (as shown in [@lst:controller-setup]). 288 | This cache is injected into the object controller instead of the unfiltered cache. 289 | With this, the sharded controller automatically uses the filtered cache as desired while it is still able to read other non-sharded objects. 290 | 291 | Furthermore, the builder makes sure that the controller respects the shard assignments and acknowledges drain operations. 292 | This is done by wrapping the actual reconciler with another generic reconciler. 293 | On each reconciliation, it checks if the object is still assigned to the respective shard by reading the `shard` label. 294 | If the object is not assigned to the shard anymore, it is discarded. 295 | Before delegating reconciliation requests to the actual reconciler, it also checks the `drain` label. 296 | In case the `drain` label is present, the controller removes both the `shard` and `drain` labels from the object and stops reconciling it. 297 | Lastly, the controller's predicates are adjusted to immediately react to events in which the object has the `drain` label. 298 | This effectively allows setting up a sharded controller without changes to the controller's code. 299 | 300 | ```go 301 | func (r *Reconciler) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { 302 | // read object from cache... 303 | labels := obj.GetLabels() 304 | if shard, ok := labels[sharding.ShardLabel]; !ok || shard != r.ShardID { 305 | // forget object 306 | return reconcile.Result{}, nil 307 | } 308 | 309 | if _, drain := labels[sharding.DrainLabel]; drain { 310 | // acknowledge drain operation 311 | patch := client.MergeFromWithOptions(obj.DeepCopyObject().(client.Object), client.MergeFromWithOptimisticLock{}) 312 | delete(labels, sharding.ShardLabel) 313 | delete(labels, sharding.DrainLabel) 314 | 315 | if err := r.client.Patch(ctx, obj, patch); err != nil { 316 | return reconcile.Result{}, fmt.Errorf("error draining object: %w", err) 317 | } 318 | 319 | // forget object 320 | return reconcile.Result{}, nil 321 | } 322 | 323 | // we are responsible, reconcile object 324 | return r.Do.Reconcile(ctx, request) 325 | } 326 | ``` 327 | 328 | : Wrapping reconciler for sharded controllers (simplified) {#lst:wrapping-reconciler} 329 | --------------------------------------------------------------------------------