├── .gitignore ├── Dockerfile ├── LICENSE ├── Makefile ├── PROJECT ├── README.md ├── bundle.Dockerfile ├── bundle ├── manifests │ ├── apps.redhat.com_clusterimpairments.yaml │ ├── cluster-impairment-operator-controller-manager-metrics-service_v1_service.yaml │ ├── cluster-impairment-operator-manager-config_v1_configmap.yaml │ ├── cluster-impairment-operator-metrics-reader_rbac.authorization.k8s.io_v1_clusterrole.yaml │ ├── cluster-impairment-operator.clusterserviceversion.yaml │ └── network-access-sa_v1_serviceaccount.yaml ├── metadata │ └── annotations.yaml └── tests │ └── scorecard │ └── config.yaml ├── config ├── crd │ ├── bases │ │ └── apps.redhat.com_clusterimpairments.yaml │ └── kustomization.yaml ├── default │ └── kustomization.yaml ├── manager │ ├── controller_manager_config.yaml │ ├── kustomization.yaml │ └── manager.yaml ├── manifests │ ├── bases │ │ └── cluster-impairment-operator.clusterserviceversion.yaml │ └── kustomization.yaml ├── namespaced │ ├── kustomization.yaml │ ├── manager_auth_proxy_patch.yaml │ └── manager_config_patch.yaml ├── network-access │ ├── kustomization.yaml │ ├── namespace.yaml │ ├── network_scc.yaml │ └── network_service_account.yaml ├── prometheus │ ├── kustomization.yaml │ └── monitor.yaml ├── rbac │ ├── auth_proxy_client_clusterrole.yaml │ ├── auth_proxy_role.yaml │ ├── auth_proxy_role_binding.yaml │ ├── auth_proxy_service.yaml │ ├── cluster-impairment_editor_role.yaml │ ├── cluster-impairment_viewer_role.yaml │ ├── clusterimpairment_editor_role.yaml │ ├── clusterimpairment_viewer_role.yaml │ ├── kustomization.yaml │ ├── leader_election_role.yaml │ ├── leader_election_role_binding.yaml │ ├── role.yaml │ ├── role_binding.yaml │ └── service_account.yaml ├── samples │ ├── apps_v1alpha1_clusterimpairment.yaml │ └── kustomization.yaml ├── scorecard │ ├── bases │ │ └── config.yaml │ ├── kustomization.yaml │ └── patches │ │ ├── basic.config.yaml │ │ └── olm.config.yaml └── testing │ ├── debug_logs_patch.yaml │ ├── kustomization.yaml │ ├── manager_image.yaml │ └── pull_policy │ ├── Always.yaml │ ├── IfNotPresent.yaml │ └── Never.yaml ├── container ├── Containerfile └── apply-impairments.py ├── manifests ├── example-1-uneven-latency-cr.yaml ├── example-2-two-min-flap-cr.yaml ├── example-3-typical-scenario-cr.yaml ├── example-4-all-impairments-cr.yaml ├── example-5-duplication.yaml ├── example-impairment-cr.yaml ├── impairment-daemonset.yaml ├── impairment-pod.yaml └── serviceaccount.yaml ├── media ├── clusterimpairment_example_1.png ├── icon.png └── simple-example-1.gif ├── molecule ├── default │ ├── converge.yml │ ├── create.yml │ ├── destroy.yml │ ├── kustomize.yml │ ├── molecule.yml │ ├── prepare.yml │ ├── tasks │ │ ├── cluster-impairment_test.yml │ │ └── clusterimpairment_test.yml │ └── verify.yml └── kind │ ├── converge.yml │ ├── create.yml │ ├── destroy.yml │ └── molecule.yml ├── operatorhub_description.txt ├── playbooks ├── .placeholder ├── clusterimpairment.yml └── roles │ ├── .placeholder │ └── deploy-impairment-workers │ ├── .travis.yml │ ├── README.md │ ├── defaults │ └── main.yml │ ├── handlers │ └── main.yml │ ├── meta │ └── main.yml │ ├── tasks │ └── main.yml │ ├── templates │ └── impairment-daemonset.yaml.j2 │ ├── tests │ ├── inventory │ └── test.yml │ └── vars │ └── main.yml ├── requirements.yml └── watches.yaml /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | # Binaries for programs and plugins 3 | *.exe 4 | *.exe~ 5 | *.dll 6 | *.so 7 | *.dylib 8 | bin 9 | 10 | # editor and IDE paraphernalia 11 | .idea 12 | *.swp 13 | *.swo 14 | *~ 15 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM quay.io/operator-framework/ansible-operator:v1.8.0 2 | ARG worker_img=quay.io/redhat-performance/cluster-impairment-worker:latest 3 | 4 | COPY requirements.yml ${HOME}/requirements.yml 5 | RUN ansible-galaxy collection install -r ${HOME}/requirements.yml \ 6 | && chmod -R ug+rwx ${HOME}/.ansible 7 | 8 | COPY watches.yaml ${HOME}/watches.yaml 9 | COPY roles/ ${HOME}/roles/ 10 | COPY playbooks/ ${HOME}/playbooks/ 11 | ENV WORKER_IMG ${worker_img} 12 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # VERSION defines the project version for the bundle. 2 | # Update this value when you upgrade the version of your project. 3 | # To re-generate a bundle for another specific version without changing the standard setup, you can: 4 | # - use the VERSION as arg of the bundle target (e.g make bundle VERSION=0.0.2) 5 | # - use environment variables to overwrite this value (e.g export VERSION=0.0.2) 6 | VERSION ?= 1.0.4 7 | 8 | # CHANNELS define the bundle channels used in the bundle. 9 | # Add a new line here if you would like to change its default config. (E.g CHANNELS = "preview,fast,stable") 10 | # To re-generate a bundle for other specific channels without changing the standard setup, you can: 11 | # - use the CHANNELS as arg of the bundle target (e.g make bundle CHANNELS=preview,fast,stable) 12 | # - use environment variables to overwrite this value (e.g export CHANNELS="preview,fast,stable") 13 | ifneq ($(origin CHANNELS), undefined) 14 | BUNDLE_CHANNELS := --channels=$(CHANNELS) 15 | else 16 | BUNDLE_CHANNELS := --channels=beta 17 | endif 18 | 19 | # DEFAULT_CHANNEL defines the default channel used in the bundle. 20 | # Add a new line here if you would like to change its default config. (E.g DEFAULT_CHANNEL = "stable") 21 | # To re-generate a bundle for any other default channel without changing the default setup, you can: 22 | # - use the DEFAULT_CHANNEL as arg of the bundle target (e.g make bundle DEFAULT_CHANNEL=stable) 23 | # - use environment variables to overwrite this value (e.g export DEFAULT_CHANNEL="stable") 24 | ifneq ($(origin DEFAULT_CHANNEL), undefined) 25 | BUNDLE_DEFAULT_CHANNEL := --default-channel=$(DEFAULT_CHANNEL) 26 | endif 27 | BUNDLE_METADATA_OPTS ?= $(BUNDLE_CHANNELS) $(BUNDLE_DEFAULT_CHANNEL) 28 | 29 | USER_ORG ?= redhat-performance 30 | 31 | # BUNDLE_IMG defines the image:tag used for the bundle. 32 | # You can use it as an arg. (E.g make bundle-build BUNDLE_IMG=/:) 33 | BUNDLE_IMG ?= quay.io/${USER_ORG}/cluster-impairment-operator-bundle:${VERSION} 34 | 35 | # The image for the worker 36 | WORKER_IMG ?= quay.io/${USER_ORG}/cluster-impairment-worker:${VERSION} 37 | 38 | # Image URL to use all building/pushing image targets 39 | IMG ?= quay.io/${USER_ORG}/cluster-impairment-operator:${VERSION} 40 | 41 | all: docker-build 42 | 43 | ##@ General 44 | 45 | # The help target prints out all targets with their descriptions organized 46 | # beneath their categories. The categories are represented by '##@' and the 47 | # target descriptions by '##'. The awk commands is responsible for reading the 48 | # entire set of makefiles included in this invocation, looking for lines of the 49 | # file as xyz: ## something, and then pretty-format the target and help. Then, 50 | # if there's a line with ##@ something, that gets pretty-printed as a category. 51 | # More info on the usage of ANSI control characters for terminal formatting: 52 | # https://en.wikipedia.org/wiki/ANSI_escape_code#SGR_parameters 53 | # More info on the awk command: 54 | # http://linuxcommand.org/lc3_adv_awk.php 55 | 56 | help: ## Display this help. 57 | @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) 58 | 59 | ##@ Build 60 | 61 | run: ansible-operator ## Run against the configured Kubernetes cluster in ~/.kube/config 62 | $(ANSIBLE_OPERATOR) run 63 | 64 | docker-build: ## Build docker image with the manager. 65 | podman build -t ${IMG} --build-arg worker_img=${WORKER_IMG} . && cd container && podman build -t ${WORKER_IMG} . 66 | 67 | docker-push: ## Push docker image with the manager. 68 | podman push ${IMG} && podman push ${WORKER_IMG} 69 | # echo 'Pushing to ${IMG}' 70 | 71 | ##@ Deployment 72 | 73 | install: kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config. 74 | $(KUSTOMIZE) build config/crd | kubectl apply -f - 75 | 76 | uninstall: kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. 77 | $(KUSTOMIZE) build config/crd | kubectl delete -f - 78 | 79 | deploy: kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config. 80 | cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} 81 | $(KUSTOMIZE) build config/default | kubectl apply -f - 82 | 83 | undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config. 84 | $(KUSTOMIZE) build config/default | kubectl delete -f - 85 | 86 | OS := $(shell uname -s | tr '[:upper:]' '[:lower:]') 87 | ARCH := $(shell uname -m | sed 's/x86_64/amd64/') 88 | 89 | .PHONY: kustomize 90 | KUSTOMIZE = $(shell pwd)/bin/kustomize 91 | kustomize: ## Download kustomize locally if necessary. 92 | ifeq (,$(wildcard $(KUSTOMIZE))) 93 | ifeq (,$(shell which kustomize 2>/dev/null)) 94 | @{ \ 95 | set -e ;\ 96 | mkdir -p $(dir $(KUSTOMIZE)) ;\ 97 | curl -sSLo - https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize/v3.5.4/kustomize_v3.5.4_$(OS)_$(ARCH).tar.gz | \ 98 | tar xzf - -C bin/ ;\ 99 | } 100 | else 101 | KUSTOMIZE = $(shell which kustomize) 102 | endif 103 | endif 104 | 105 | .PHONY: ansible-operator 106 | ANSIBLE_OPERATOR = $(shell pwd)/bin/ansible-operator 107 | ansible-operator: ## Download ansible-operator locally if necessary, preferring the $(pwd)/bin path over global if both exist. 108 | ifeq (,$(wildcard $(ANSIBLE_OPERATOR))) 109 | ifeq (,$(shell which ansible-operator 2>/dev/null)) 110 | @{ \ 111 | set -e ;\ 112 | mkdir -p $(dir $(ANSIBLE_OPERATOR)) ;\ 113 | curl -sSLo $(ANSIBLE_OPERATOR) https://github.com/operator-framework/operator-sdk/releases/download/v1.8.0/ansible-operator_$(OS)_$(ARCH) ;\ 114 | chmod +x $(ANSIBLE_OPERATOR) ;\ 115 | } 116 | else 117 | ANSIBLE_OPERATOR = $(shell which ansible-operator) 118 | endif 119 | endif 120 | 121 | .PHONY: bundle 122 | bundle: kustomize ## Generate bundle manifests and metadata, then validate generated files. 123 | operator-sdk generate kustomize manifests -q 124 | cd config/manager && $(KUSTOMIZE) edit set image controller=$(IMG) 125 | $(KUSTOMIZE) build config/manifests | operator-sdk generate bundle --overwrite --version $(VERSION) $(BUNDLE_METADATA_OPTS) 126 | operator-sdk bundle validate ./bundle 127 | 128 | .PHONY: bundle-build 129 | bundle-build: ## Build the bundle image. 130 | podman build -f bundle.Dockerfile -t $(BUNDLE_IMG) . 131 | 132 | .PHONY: bundle-push 133 | bundle-push: ## Push the bundle image. 134 | $(MAKE) docker-push IMG=$(BUNDLE_IMG) 135 | 136 | .PHONY: opm 137 | OPM = ./bin/opm 138 | opm: ## Download opm locally if necessary. 139 | ifeq (,$(wildcard $(OPM))) 140 | ifeq (,$(shell which opm 2>/dev/null)) 141 | @{ \ 142 | set -e ;\ 143 | mkdir -p $(dir $(OPM)) ;\ 144 | curl -sSLo $(OPM) https://github.com/operator-framework/operator-registry/releases/download/v1.15.1/$(OS)-$(ARCH)-opm ;\ 145 | chmod +x $(OPM) ;\ 146 | } 147 | else 148 | OPM = $(shell which opm) 149 | endif 150 | endif 151 | 152 | # A comma-separated list of bundle images (e.g. make catalog-build BUNDLE_IMGS=example.com/operator-bundle:v0.1.0,example.com/operator-bundle:v0.2.0). 153 | # These images MUST exist in a registry and be pull-able. 154 | BUNDLE_IMGS ?= $(BUNDLE_IMG) 155 | 156 | # The image tag given to the resulting catalog image (e.g. make catalog-build CATALOG_IMG=example.com/operator-catalog:v0.2.0). 157 | CATALOG_IMG ?= $(IMAGE_TAG_BASE)-catalog:v$(VERSION) 158 | 159 | # Set CATALOG_BASE_IMG to an existing catalog image tag to add $BUNDLE_IMGS to that image. 160 | ifneq ($(origin CATALOG_BASE_IMG), undefined) 161 | FROM_INDEX_OPT := --from-index $(CATALOG_BASE_IMG) 162 | endif 163 | 164 | # Build a catalog image by adding bundle images to an empty catalog using the operator package manager tool, 'opm'. 165 | # This recipe invokes 'opm' in 'semver' bundle add mode. For more information on add modes, see: 166 | # https://github.com/operator-framework/community-operators/blob/7f1438c/docs/packaging-operator.md#updating-your-existing-operator 167 | .PHONY: catalog-build 168 | catalog-build: opm ## Build a catalog image. 169 | $(OPM) index add --container-tool podman --mode semver --tag $(CATALOG_IMG) --bundles $(BUNDLE_IMGS) $(FROM_INDEX_OPT) 170 | 171 | # Push the catalog image. 172 | .PHONY: catalog-push 173 | catalog-push: ## Push a catalog image. 174 | $(MAKE) docker-push IMG=$(CATALOG_IMG) 175 | -------------------------------------------------------------------------------- /PROJECT: -------------------------------------------------------------------------------- 1 | domain: redhat.com 2 | layout: 3 | - ansible.sdk.operatorframework.io/v1 4 | plugins: 5 | manifests.sdk.operatorframework.io/v2: {} 6 | scorecard.sdk.operatorframework.io/v2: {} 7 | projectName: cluster-impairment-operator 8 | resources: 9 | - api: 10 | crdVersion: v1 11 | namespaced: true 12 | domain: redhat.com 13 | group: apps 14 | kind: ClusterImpairment 15 | version: v1alpha1 16 | version: "3" 17 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # cluster-impairment-operator 2 | 3 | cluster-impairment-operator is an operator designed to automate the application of impairments to the nodes in a cluster. 4 | 5 | ![Demo](media/simple-example-1.gif) 6 | 7 | ## Features 8 | 9 | Traffic Direction: 10 | * Egress impairments affect the traffic going out. 11 | * Ingress impairments affect the traffic coming in. 12 | * Ingress impairments require the kernel module IFB. 13 | 14 | ### Impairments 15 | 16 | | Impairment | Description | Unit | Uses Netem | Correlation Supported | 17 | |---------------|--------------------------------------------|---------|------------|-----------------------| 18 | | Bandwidth | The bandwidth limit | kbit/s | Yes | No | 19 | | Latency | The delay of the packets | ms | Yes | Yes | 20 | | Packet Loss | The percent of packets that are dropped | percent | Yes | Yes | 21 | | Corruption | The percent of packets that are corrupted | percent | Yes | Yes | 22 | | Duplication | The percent of packets that are duplicated | percent | Yes | Yes | 23 | | Link Flapping | Turns the interface on and off | bool | No | No | 24 | 25 | On the tested environment (RHEL CoreOS 48.84), the impairments can be used alongside link flapping. 26 | 27 | #### Latency Options 28 | 29 | In addition to simply delaying the packets, there are advanced latency options. 30 | 31 | **Jitter** 32 | 33 | Semi-randomly adds or subtracts from the latency according to the distribution up to the amount specified. 34 | If latency is 100ms, and jitter is 10ms, the actual latency will vary 100±10ms. 35 | 36 | 37 | **Distribution** 38 | 39 | The distribution of the jitter. The options are: 40 | * Normal 41 | * Uniform 42 | * Pareto 43 | * Paretonormal 44 | 45 | **Jitter correlation** 46 | 47 | The percent chance that the next latency's jitter value will correlate with the preceeding latency. 48 | 49 | **Reorder** 50 | 51 | The percentage of packets that are not delayed, causing a reorder due to them being sent before the delayed ones. 52 | Applying jitter itself has the potential to also cause reordering. 53 | 54 | **Reorder correlation** 55 | 56 | The percet chance that the value for the next reorder will correlate with the preceeding value. 57 | 58 | #### Loss options 59 | 60 | **Correlation** 61 | 62 | The percent chance that the previous loss value for a packet correlates with the loss value for the next packet. 63 | 64 | #### Corruption options 65 | 66 | **Corruption** 67 | 68 | The percent chance that each packet will be corrupted. 69 | 70 | **Corruption correlation** 71 | 72 | The percet chance that the value for the next corruption will correlate with the preceeding value. 73 | 74 | #### Duplication options 75 | 76 | **Duplication** 77 | 78 | The percent chance that each packet will be duplicated. 79 | 80 | **Duplication correlation** 81 | 82 | The percet chance that the value for the next duplication will correlate with the preceeding value. 83 | 84 | 85 | ## Configuration 86 | 87 | Here is an example of the ClusterImpairment custom resource. 88 | ```yaml 89 | apiVersion: apps.redhat.com/v1alpha1 90 | kind: ClusterImpairment 91 | metadata: 92 | name: test-impairment-cr 93 | spec: 94 | duration: 30 # seconds 95 | startDelay: 5 # seconds. It typically takes about 2-3 seconds for the Daemonset to run 96 | interfaces: 97 | - "ens3" 98 | ingress: # uses ifb 99 | bandwidth: 0 # kbit 100 | latency: 10 # ms 101 | latencyOptions: 102 | jitter: 5 # ms 103 | jitterCorrelation: 25 # percent 104 | distribution: normal 105 | reorder: 25 # percent of packets that will skip the delay 106 | reorderCorrelation: 25 # percent 107 | loss: 0 # percent 108 | lossOptions: 109 | correlation: 25 # percent 110 | corruption: 0.1 # percent 111 | corruptionOptions: 112 | correlation: 25 # percent 113 | duplication: 1 # percent 114 | duplicationOptions: 115 | correlation: 25 # percent 116 | egress: 117 | bandwidth: 0 # kbit 118 | latency: 100 # ms 119 | loss: 0 # percent 120 | linkFlapping: 121 | enable: false 122 | downTime: 3 # Seconds 123 | upTime: 3 # Seconds 124 | nodeSelector: 125 | key: "node-role.kubernetes.io/worker" 126 | value: "" 127 | ``` 128 | 129 | #### Interfaces 130 | 131 | The interfaces configuration option is a list of all interfaces that the impairments should be applied to. The current implementation will error out once it hits an invalid interface. 132 | 133 | If an invalid interface is found, it will print out the list of interfaces. 134 | 135 | #### Node Selector 136 | 137 | The default node selector is all worker nodes, but you can change it to whatever node selector you want by setting the key and value. 138 | There is a toleration to allow it to run on master nodes if the node selector includes master nodes. 139 | 140 | #### Duration 141 | 142 | The number of seconds the impairments should run. It will try to sync the start and end time between all pods. 143 | If set to -1, it will run indefinitely (a year), until you remove the ClusterImpairment custom resource. This is good for testing that requires steady impairments. 144 | 145 | If the script is link flapping, set this value to be short enough so that if the link flap interrupts communication between the nodes, the daemonset will remove itself. 146 | 147 | #### Start Delay 148 | 149 | The delay before starting the script. If you want the pods to be in sync, a minimum of a few seconds should be used because the pods take time to start up. 150 | 151 | You can also utilize this feature to run an separate impairment after another. Just apply two resources at the same time, but add the duration and start delay of the first to the start delay of the second. 152 | 153 | #### Ingress and Egress 154 | 155 | The configuration sections "ingress" and "egress" apply to each direction. 156 | 157 | ##### Examples: 158 | 159 | **Example 1** 160 | In this example, egress latency is set to 100ms, but the ingress latency is set to 10ms. So the latency to the interface will end up being 10ms, but 100ms going out. When pinging, this will result in 110ms of latency. 161 | 162 | ![diagram](media/clusterimpairment_example_1.png) 163 | 164 | ```yaml 165 | apiVersion: apps.redhat.com/v1alpha1 166 | kind: ClusterImpairment 167 | metadata: 168 | name: uneven-latency 169 | spec: 170 | duration: 60 171 | startDelay: 5 172 | interfaces: 173 | - "ens3" 174 | ingress: 175 | latency: 10 # ms 176 | egress: 177 | latency: 100 # ms 178 | ``` 179 | 180 | **Example 2** 181 | In this example, link flapping will be enabled, and it will turn the interface on and off every 2 minutes. That is long enough for kubernetes to determine that a node is unavailable. 182 | 183 | ```yaml 184 | apiVersion: apps.redhat.com/v1alpha1 185 | kind: ClusterImpairment 186 | metadata: 187 | name: two-min-flap 188 | spec: 189 | duration: 480 190 | startDelay: 5 191 | interfaces: 192 | - "ens3" 193 | linkFlapping: 194 | enable: true 195 | downTime: 120 # Seconds 196 | upTime: 120 # Seconds 197 | ``` 198 | 199 | **Example 3** 200 | In this example, a realistic set of impairments is applied to `ens3` and `eno1` for 30 seconds: 201 | 202 | ```yaml 203 | apiVersion: apps.redhat.com/v1alpha1 204 | kind: ClusterImpairment 205 | metadata: 206 | name: typical-scenario 207 | spec: 208 | duration: 30 # seconds 209 | startDelay: 5 # seconds 210 | interfaces: 211 | - "ens3" 212 | - "eno1" 213 | egress: 214 | latency: 50 # ms. Bidirectional, so total of 100ms 215 | loss: 0.01 # percent 216 | ingress: 217 | latency: 50 # ms. Bidirectional, so total of 100ms 218 | loss: 0.01 # percent 219 | ``` 220 | 221 | **Example 4** 222 | In this example, a combination of latency, packet loss, bandwidth, and link flapping is applied. 223 | ```yaml 224 | apiVersion: apps.redhat.com/v1alpha1 225 | kind: ClusterImpairment 226 | metadata: 227 | name: all-impairments 228 | spec: 229 | duration: 480 # seconds 230 | startDelay: 5 # seconds 231 | interfaces: 232 | - "ens3" 233 | egress: 234 | latency: 50 # ms. Bidirectional, so total of 100ms 235 | loss: 0.02 # percent 236 | bandwidth: 1000 # 1000 kbit/s, about 1 mbit/s 237 | ingress: 238 | latency: 50 # ms. Bidirectional, so total of 100ms 239 | loss: 0.02 # percent 240 | bandwidth: 1000 # 1000 kbit/s, about 1 mbit/s 241 | linkFlapping: 242 | enable: true 243 | downTime: 30 # Seconds 244 | upTime: 120 # Seconds 245 | ``` 246 | 247 | **Example 5** 248 | In this example, the packets are duplicated 25% of the time on ingress. In real situations it will be a very small percent; less than one percent. But this will clearly show up in ping. 249 | 250 | ```yaml 251 | apiVersion: apps.redhat.com/v1alpha1 252 | kind: ClusterImpairment 253 | metadata: 254 | name: duplication 255 | spec: 256 | duration: 60 257 | startDelay: 5 258 | interfaces: 259 | - "ens3" 260 | ingress: 261 | duplication: 25 # percent 262 | ``` 263 | 264 | 265 | ## Setup 266 | 267 | ### Requirements 268 | 269 | 1. You need `make` installed. 270 | 2. You need access to the kubernetes cluster with a kubeconfig. 271 | 272 | ### Installation 273 | 274 | To run using the current latest image: 275 | 1. Clone the repository. 276 | 2. Run `make deploy` with KUBECONFIG environment variable set to the location of the kubeconfig. 277 | 278 | To run with your own code, there are more steps. 279 | 280 | 1. Fork the repository. 281 | 2. Clone to a machine that has access to the Kubernetes cluster and the kubeconfig. 282 | 3. Modify the makefile to change the `IMG` variable to your image repository. If you do not have podman installed, also change podman to docker. 283 | 4. Run `make docker-build` then `make docker-push`. 284 | 5. Then run `make deploy` 285 | 286 | ### Deploying from operator-hub 287 | 288 | See the instructions on https://operatorhub.io/operator/cluster-impairment-operator 289 | 290 | ## Running impairments 291 | 292 | First, configure a ClusterImpairment type's spec with the information for the impairment you want to run. Most importantly, set the interface(s). You can list the interfaces with `ifconfig`. If an invalid interface is set, the pod's logs will show `ifconfig` output. 293 | 294 | Once the clusterimpairment type is set, apply it and it will work. 295 | 296 | To re-run an impairment, delete the impairment with `oc delete -f path/to/manifest.yaml`, then apply it again. 297 | 298 | ## Limitations 299 | 300 | ### Multiple Impairments 301 | 302 | You should avoid any configurations that apply impairments to the same interface on the same node. There are potential conflicts. The worker pod will attempt to remove all impairments on that interface before applying new ones. 303 | 304 | Instead, take advantage of the full control of both ingress and egress impairments from within the same ClusterImpairment resource. 305 | 306 | There is currently a configured limit of 20 ClusterImpairment resources working at the same time. If that is a limitation for your use case, you can change it in `config/manager/manager.yaml` or you can try to find a way to make each ClusterImpairment do more. 307 | 308 | ### Traffic Control (TC) 309 | 310 | Traffic control is how cluster-impairment-operator applies the latency, bandwidth, and packet loss impairments. The limitation is due to Linux not being realtime operating system, so the impairment will not be perfectly consistent. 311 | 312 | ### Link Flapping 313 | 314 | When link flapping, if you flap the link that Kubernetes uses to communicate with the pods, you may be unable to remove the pod until the link is on long enough for Kubernetes to communicate with the impaired node. 315 | 316 | In this case, it is helpful to set the duration properly instead of running for an indefinite or large amount of time, because the node will properly go back to the unimpaired state at that time. 317 | 318 | If the cluster becomes offline due to the link flapping when you do not want it to be offline, soft restarting the nodes after removing the custom resource should remove all impairments. 319 | -------------------------------------------------------------------------------- /bundle.Dockerfile: -------------------------------------------------------------------------------- 1 | FROM scratch 2 | 3 | # Core bundle labels. 4 | LABEL operators.operatorframework.io.bundle.mediatype.v1=registry+v1 5 | LABEL operators.operatorframework.io.bundle.manifests.v1=manifests/ 6 | LABEL operators.operatorframework.io.bundle.metadata.v1=metadata/ 7 | LABEL operators.operatorframework.io.bundle.package.v1=cluster-impairment-operator 8 | LABEL operators.operatorframework.io.bundle.channels.v1=beta 9 | LABEL operators.operatorframework.io.metrics.builder=operator-sdk-v1.10.0+git 10 | LABEL operators.operatorframework.io.metrics.mediatype.v1=metrics+v1 11 | LABEL operators.operatorframework.io.metrics.project_layout=ansible.sdk.operatorframework.io/v1 12 | 13 | # Labels for testing. 14 | LABEL operators.operatorframework.io.test.mediatype.v1=scorecard+v1 15 | LABEL operators.operatorframework.io.test.config.v1=tests/scorecard/ 16 | 17 | # Copy files to locations specified by labels. 18 | COPY bundle/manifests /manifests/ 19 | COPY bundle/metadata /metadata/ 20 | COPY bundle/tests/scorecard /tests/scorecard/ 21 | -------------------------------------------------------------------------------- /bundle/manifests/apps.redhat.com_clusterimpairments.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apiextensions.k8s.io/v1 2 | kind: CustomResourceDefinition 3 | metadata: 4 | creationTimestamp: null 5 | name: clusterimpairments.apps.redhat.com 6 | spec: 7 | group: apps.redhat.com 8 | names: 9 | kind: ClusterImpairment 10 | listKind: ClusterImpairmentList 11 | plural: clusterimpairments 12 | singular: clusterimpairment 13 | scope: Cluster 14 | versions: 15 | - additionalPrinterColumns: 16 | - description: Run status 17 | jsonPath: .status.run_status 18 | name: Status 19 | type: string 20 | - description: Run status details 21 | jsonPath: .status.status_details 22 | name: Details 23 | type: string 24 | name: v1alpha1 25 | schema: 26 | openAPIV3Schema: 27 | description: ClusterImpairment is the Schema for the clusterimpairments API 28 | properties: 29 | apiVersion: 30 | description: 'APIVersion defines the versioned schema of this representation 31 | of an object. Servers should convert recognized schemas to the latest 32 | internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' 33 | type: string 34 | kind: 35 | description: 'Kind is a string value representing the REST resource this 36 | object represents. Servers may infer this from the endpoint the client 37 | submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' 38 | type: string 39 | metadata: 40 | type: object 41 | spec: 42 | description: Spec defines the desired state of ClusterImpairment 43 | properties: 44 | duration: 45 | description: The duration of the impairment in seconds. 46 | type: integer 47 | egress: 48 | description: The configuration section that specifies the egress impairments. 49 | properties: 50 | bandwidth: 51 | description: The bandwidth limit in kbit/sec 52 | type: integer 53 | corruption: 54 | description: The percent of packets that are corrupted 55 | type: number 56 | corruptionOptions: 57 | description: Advanced corruption options 58 | properties: 59 | correlation: 60 | description: The correlation between sequential corruption 61 | values 62 | type: number 63 | type: object 64 | duplication: 65 | description: The percent of packets duplicated 66 | type: number 67 | duplicationOptions: 68 | description: Advanced duplication options 69 | properties: 70 | correlation: 71 | description: The correlation between sequential duplication 72 | values 73 | type: number 74 | type: object 75 | latency: 76 | description: The latency applied in ms 77 | type: number 78 | latencyOptions: 79 | description: 'Advanced latency options. Example: jitter' 80 | properties: 81 | distribution: 82 | description: 'The way the jitter is distributed. Options: 83 | Normal, Uniform, Pareto, Paretonormal' 84 | type: string 85 | jitter: 86 | description: Variation in the latency that follows the specified 87 | distribution. 88 | type: number 89 | jitterCorrelation: 90 | description: The correlation between sequential jitter values 91 | type: number 92 | reorder: 93 | description: The percentage of packets that are not delayed, 94 | causing reordering 95 | type: number 96 | reorderCorrelation: 97 | description: The correlation between sequential reorder values 98 | type: number 99 | type: object 100 | loss: 101 | description: The packet loss in percent 102 | type: number 103 | lossOptions: 104 | description: Advanced packet loss options 105 | properties: 106 | correlation: 107 | description: The correlation between sequential packet loss 108 | values 109 | type: number 110 | type: object 111 | type: object 112 | ingress: 113 | description: The configuration section that specifies the ingress 114 | impairments. 115 | properties: 116 | bandwidth: 117 | description: The bandwidth limit in kbit/sec 118 | type: integer 119 | corruption: 120 | description: The percent of packets that are corrupted 121 | type: number 122 | corruptionOptions: 123 | description: Advanced corruption options 124 | properties: 125 | correlation: 126 | description: The correlation between sequential corruption 127 | values 128 | type: number 129 | type: object 130 | duplication: 131 | description: The percent of packets duplicated 132 | type: number 133 | duplicationOptions: 134 | description: Advanced duplication options 135 | properties: 136 | correlation: 137 | description: The correlation between sequential duplication 138 | values 139 | type: number 140 | type: object 141 | latency: 142 | description: The latency applied in ms 143 | type: number 144 | latencyOptions: 145 | description: 'Advanced latency options. Example: jitter' 146 | properties: 147 | distribution: 148 | description: 'The way the jitter is distributed. Options: 149 | Normal, Uniform, Pareto, Paretonormal' 150 | type: string 151 | jitter: 152 | description: Variation in the latency that follows the specified 153 | distribution. 154 | type: number 155 | jitterCorrelation: 156 | description: The correlation between sequential jitter values 157 | type: number 158 | reorder: 159 | description: The percentage of packets that are not delayed, 160 | causing reordering 161 | type: number 162 | reorderCorrelation: 163 | description: The correlation between sequential reorder values 164 | type: number 165 | type: object 166 | loss: 167 | description: The packet loss in percent 168 | type: number 169 | lossOptions: 170 | description: Advanced packet loss options 171 | properties: 172 | correlation: 173 | description: The correlation between sequential packet loss 174 | values 175 | type: number 176 | type: object 177 | type: object 178 | interfaces: 179 | description: All interfaces that the impairments should be applied 180 | to. Must be valid interfaces or the impairments will fail to apply. 181 | items: 182 | type: string 183 | type: array 184 | linkFlapping: 185 | description: The configuration section that specifies the link flapping 186 | impairments. 187 | properties: 188 | downTime: 189 | description: The duration that the link should be disabled. 190 | type: integer 191 | enable: 192 | description: Whether to enable link flapping. 193 | type: boolean 194 | upTime: 195 | description: The duration that the link should be enabled. 196 | type: integer 197 | type: object 198 | nodeSelector: 199 | description: 'The configuration section that specifies the node selector 200 | that should be applied to the daemonset. Default: worker nodes.' 201 | properties: 202 | key: 203 | description: The key for the node selector 204 | type: string 205 | value: 206 | description: The value for the node selector 207 | type: string 208 | type: object 209 | startDelay: 210 | description: The delay (in seconds) before starting the impairments. 211 | At least 5 seconds recommended for Kubernetes and for synchronization 212 | of the impairments. 213 | type: integer 214 | type: object 215 | status: 216 | description: Status defines the observed state of ClusterImpairment 217 | type: object 218 | x-kubernetes-preserve-unknown-fields: true 219 | type: object 220 | served: true 221 | storage: true 222 | subresources: 223 | status: {} 224 | status: 225 | acceptedNames: 226 | kind: "" 227 | plural: "" 228 | conditions: null 229 | storedVersions: null 230 | -------------------------------------------------------------------------------- /bundle/manifests/cluster-impairment-operator-controller-manager-metrics-service_v1_service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | creationTimestamp: null 5 | labels: 6 | control-plane: controller-manager 7 | name: cluster-impairment-operator-controller-manager-metrics-service 8 | spec: 9 | ports: 10 | - name: https 11 | port: 8443 12 | targetPort: https 13 | selector: 14 | control-plane: controller-manager 15 | status: 16 | loadBalancer: {} 17 | -------------------------------------------------------------------------------- /bundle/manifests/cluster-impairment-operator-manager-config_v1_configmap.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | data: 3 | controller_manager_config.yaml: | 4 | apiVersion: controller-runtime.sigs.k8s.io/v1alpha1 5 | kind: ControllerManagerConfig 6 | health: 7 | healthProbeBindAddress: :6789 8 | metrics: 9 | bindAddress: 127.0.0.1:8080 10 | leaderElection: 11 | leaderElect: true 12 | resourceName: 811c9dc5.redhat.com 13 | kind: ConfigMap 14 | metadata: 15 | name: cluster-impairment-operator-manager-config 16 | -------------------------------------------------------------------------------- /bundle/manifests/cluster-impairment-operator-metrics-reader_rbac.authorization.k8s.io_v1_clusterrole.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRole 3 | metadata: 4 | creationTimestamp: null 5 | name: cluster-impairment-operator-metrics-reader 6 | rules: 7 | - nonResourceURLs: 8 | - /metrics 9 | verbs: 10 | - get 11 | -------------------------------------------------------------------------------- /bundle/manifests/network-access-sa_v1_serviceaccount.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | creationTimestamp: null 5 | name: network-access-sa 6 | -------------------------------------------------------------------------------- /bundle/metadata/annotations.yaml: -------------------------------------------------------------------------------- 1 | annotations: 2 | # Core bundle annotations. 3 | operators.operatorframework.io.bundle.mediatype.v1: registry+v1 4 | operators.operatorframework.io.bundle.manifests.v1: manifests/ 5 | operators.operatorframework.io.bundle.metadata.v1: metadata/ 6 | operators.operatorframework.io.bundle.package.v1: cluster-impairment-operator 7 | operators.operatorframework.io.bundle.channels.v1: beta 8 | operators.operatorframework.io.metrics.builder: operator-sdk-v1.10.0+git 9 | operators.operatorframework.io.metrics.mediatype.v1: metrics+v1 10 | operators.operatorframework.io.metrics.project_layout: ansible.sdk.operatorframework.io/v1 11 | 12 | # Annotations for testing. 13 | operators.operatorframework.io.test.mediatype.v1: scorecard+v1 14 | operators.operatorframework.io.test.config.v1: tests/scorecard/ 15 | -------------------------------------------------------------------------------- /bundle/tests/scorecard/config.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: scorecard.operatorframework.io/v1alpha3 2 | kind: Configuration 3 | metadata: 4 | name: config 5 | stages: 6 | - parallel: true 7 | tests: 8 | - entrypoint: 9 | - scorecard-test 10 | - basic-check-spec 11 | image: quay.io/operator-framework/scorecard-test:v1.8.0 12 | labels: 13 | suite: basic 14 | test: basic-check-spec-test 15 | - entrypoint: 16 | - scorecard-test 17 | - olm-bundle-validation 18 | image: quay.io/operator-framework/scorecard-test:v1.8.0 19 | labels: 20 | suite: olm 21 | test: olm-bundle-validation-test 22 | - entrypoint: 23 | - scorecard-test 24 | - olm-crds-have-validation 25 | image: quay.io/operator-framework/scorecard-test:v1.8.0 26 | labels: 27 | suite: olm 28 | test: olm-crds-have-validation-test 29 | - entrypoint: 30 | - scorecard-test 31 | - olm-crds-have-resources 32 | image: quay.io/operator-framework/scorecard-test:v1.8.0 33 | labels: 34 | suite: olm 35 | test: olm-crds-have-resources-test 36 | - entrypoint: 37 | - scorecard-test 38 | - olm-spec-descriptors 39 | image: quay.io/operator-framework/scorecard-test:v1.8.0 40 | labels: 41 | suite: olm 42 | test: olm-spec-descriptors-test 43 | - entrypoint: 44 | - scorecard-test 45 | - olm-status-descriptors 46 | image: quay.io/operator-framework/scorecard-test:v1.8.0 47 | labels: 48 | suite: olm 49 | test: olm-status-descriptors-test 50 | -------------------------------------------------------------------------------- /config/crd/bases/apps.redhat.com_clusterimpairments.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apiextensions.k8s.io/v1 3 | kind: CustomResourceDefinition 4 | metadata: 5 | name: clusterimpairments.apps.redhat.com 6 | spec: 7 | group: apps.redhat.com 8 | names: 9 | kind: ClusterImpairment 10 | listKind: ClusterImpairmentList 11 | plural: clusterimpairments 12 | singular: clusterimpairment 13 | scope: Cluster 14 | versions: 15 | - additionalPrinterColumns: 16 | - description: Run status 17 | jsonPath: .status.run_status 18 | name: Status 19 | type: string 20 | - description: Run status details 21 | jsonPath: .status.status_details 22 | name: Details 23 | type: string 24 | name: v1alpha1 25 | schema: 26 | openAPIV3Schema: 27 | description: ClusterImpairment is the Schema for the clusterimpairments API 28 | properties: 29 | apiVersion: 30 | description: 'APIVersion defines the versioned schema of this representation 31 | of an object. Servers should convert recognized schemas to the latest 32 | internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' 33 | type: string 34 | kind: 35 | description: 'Kind is a string value representing the REST resource this 36 | object represents. Servers may infer this from the endpoint the client 37 | submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' 38 | type: string 39 | metadata: 40 | type: object 41 | spec: 42 | description: "Spec defines the desired state of ClusterImpairment" 43 | type: object 44 | properties: 45 | duration: 46 | description: "The duration of the impairment in seconds." 47 | type: integer 48 | startDelay: 49 | description: "The delay (in seconds) before starting the impairments. At least 5 seconds recommended for Kubernetes and for synchronization of the impairments." 50 | type: integer 51 | interfaces: 52 | description: "All interfaces that the impairments should be applied to. Must be valid interfaces or the impairments will fail to apply." 53 | type: array 54 | items: 55 | type: string 56 | ingress: 57 | description: "The configuration section that specifies the ingress impairments." 58 | type: object 59 | properties: 60 | bandwidth: 61 | description: "The bandwidth limit in kbit/sec" 62 | type: integer 63 | latency: 64 | description: "The latency applied in ms" 65 | type: number 66 | latencyOptions: 67 | description: "Advanced latency options. Example: jitter" 68 | type: object 69 | properties: 70 | jitter: 71 | description: "Variation in the latency that follows the specified distribution." 72 | type: number 73 | jitterCorrelation: 74 | description: "The correlation between sequential jitter values" 75 | type: number 76 | distribution: 77 | description: "The way the jitter is distributed. Options: Normal, Uniform, Pareto, Paretonormal" 78 | type: string 79 | reorder: 80 | description: "The percentage of packets that are not delayed, causing reordering" 81 | type: number 82 | reorderCorrelation: 83 | description: "The correlation between sequential reorder values" 84 | type: number 85 | loss: 86 | description: "The packet loss in percent" 87 | type: number 88 | lossOptions: 89 | description: "Advanced packet loss options" 90 | type: object 91 | properties: 92 | correlation: 93 | type: number 94 | description: "The correlation between sequential packet loss values" 95 | corruption: 96 | description: "The percent of packets that are corrupted" 97 | type: number 98 | corruptionOptions: 99 | description: "Advanced corruption options" 100 | type: object 101 | properties: 102 | correlation: 103 | type: number 104 | description: "The correlation between sequential corruption values" 105 | duplication: 106 | description: "The percent of packets duplicated" 107 | type: number 108 | duplicationOptions: 109 | description: "Advanced duplication options" 110 | type: object 111 | properties: 112 | correlation: 113 | type: number 114 | description: "The correlation between sequential duplication values" 115 | egress: 116 | description: "The configuration section that specifies the egress impairments." 117 | type: object 118 | properties: 119 | bandwidth: 120 | description: "The bandwidth limit in kbit/sec" 121 | type: integer 122 | latency: 123 | description: "The latency applied in ms" 124 | type: number 125 | latencyOptions: 126 | description: "Advanced latency options. Example: jitter" 127 | type: object 128 | properties: 129 | jitter: 130 | description: "Variation in the latency that follows the specified distribution." 131 | type: number 132 | jitterCorrelation: 133 | description: "The correlation between sequential jitter values" 134 | type: number 135 | distribution: 136 | description: "The way the jitter is distributed. Options: Normal, Uniform, Pareto, Paretonormal" 137 | type: string 138 | reorder: 139 | description: "The percentage of packets that are not delayed, causing reordering" 140 | type: number 141 | reorderCorrelation: 142 | description: "The correlation between sequential reorder values" 143 | type: number 144 | loss: 145 | description: "The packet loss in percent" 146 | type: number 147 | lossOptions: 148 | description: "Advanced packet loss options" 149 | type: object 150 | properties: 151 | correlation: 152 | type: number 153 | description: "The correlation between sequential packet loss values" 154 | corruption: 155 | description: "The percent of packets that are corrupted" 156 | type: number 157 | corruptionOptions: 158 | description: "Advanced corruption options" 159 | type: object 160 | properties: 161 | correlation: 162 | type: number 163 | description: "The correlation between sequential corruption values" 164 | duplication: 165 | description: "The percent of packets duplicated" 166 | type: number 167 | duplicationOptions: 168 | description: "Advanced duplication options" 169 | type: object 170 | properties: 171 | correlation: 172 | type: number 173 | description: "The correlation between sequential duplication values" 174 | linkFlapping: 175 | description: "The configuration section that specifies the link flapping impairments." 176 | type: object 177 | properties: 178 | enable: 179 | description: "Whether to enable link flapping." 180 | type: boolean 181 | downTime: 182 | description: "The duration that the link should be disabled." 183 | type: integer 184 | upTime: 185 | description: "The duration that the link should be enabled." 186 | type: integer 187 | nodeSelector: 188 | description: "The configuration section that specifies the node selector that should be applied to the daemonset. Default: worker nodes." 189 | type: object 190 | properties: 191 | key: 192 | description: "The key for the node selector" 193 | type: string 194 | value: 195 | description: "The value for the node selector" 196 | type: string 197 | status: 198 | description: "Status defines the observed state of ClusterImpairment" 199 | type: object 200 | x-kubernetes-preserve-unknown-fields: true 201 | type: object 202 | served: true 203 | storage: true 204 | subresources: 205 | status: 206 | run_status: not_run 207 | -------------------------------------------------------------------------------- /config/crd/kustomization.yaml: -------------------------------------------------------------------------------- 1 | # This kustomization.yaml is not intended to be run by itself, 2 | # since it depends on service name and namespace that are out of this kustomize package. 3 | # It should be run by config/default 4 | resources: 5 | #- bases/impairment.redhat.com.redhat.com_cluster-impairments.yaml 6 | - bases/apps.redhat.com_clusterimpairments.yaml 7 | #+kubebuilder:scaffold:crdkustomizeresource 8 | -------------------------------------------------------------------------------- /config/default/kustomization.yaml: -------------------------------------------------------------------------------- 1 | bases: 2 | - ../network-access 3 | - ../namespaced 4 | -------------------------------------------------------------------------------- /config/manager/controller_manager_config.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: controller-runtime.sigs.k8s.io/v1alpha1 2 | kind: ControllerManagerConfig 3 | health: 4 | healthProbeBindAddress: :6789 5 | metrics: 6 | bindAddress: 127.0.0.1:8080 7 | 8 | leaderElection: 9 | leaderElect: true 10 | resourceName: 811c9dc5.redhat.com 11 | -------------------------------------------------------------------------------- /config/manager/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - manager.yaml 3 | 4 | generatorOptions: 5 | disableNameSuffixHash: true 6 | 7 | configMapGenerator: 8 | - files: 9 | - controller_manager_config.yaml 10 | name: manager-config 11 | apiVersion: kustomize.config.k8s.io/v1beta1 12 | kind: Kustomization 13 | images: 14 | - name: controller 15 | newName: quay.io/redhat-performance/cluster-impairment-operator 16 | newTag: 1.0.4 17 | -------------------------------------------------------------------------------- /config/manager/manager.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | labels: 5 | control-plane: controller-manager 6 | name: system 7 | --- 8 | apiVersion: apps/v1 9 | kind: Deployment 10 | metadata: 11 | name: controller-manager 12 | namespace: system 13 | labels: 14 | control-plane: controller-manager 15 | spec: 16 | selector: 17 | matchLabels: 18 | control-plane: controller-manager 19 | replicas: 1 20 | template: 21 | metadata: 22 | labels: 23 | control-plane: controller-manager 24 | spec: 25 | securityContext: 26 | runAsNonRoot: true 27 | containers: 28 | - args: 29 | - --leader-elect 30 | - --leader-election-id=cluster-impairment-operator 31 | - --max-concurrent-reconciles=20 32 | image: controller:latest 33 | name: manager 34 | env: 35 | - name: ANSIBLE_GATHERING 36 | value: explicit 37 | - name: POD_NAMESPACE 38 | valueFrom: 39 | fieldRef: 40 | fieldPath: metadata.namespace 41 | securityContext: 42 | allowPrivilegeEscalation: false 43 | livenessProbe: 44 | httpGet: 45 | path: /healthz 46 | port: 6789 47 | initialDelaySeconds: 15 48 | periodSeconds: 20 49 | readinessProbe: 50 | httpGet: 51 | path: /readyz 52 | port: 6789 53 | initialDelaySeconds: 5 54 | periodSeconds: 10 55 | serviceAccountName: controller-manager 56 | terminationGracePeriodSeconds: 10 57 | -------------------------------------------------------------------------------- /config/manifests/bases/cluster-impairment-operator.clusterserviceversion.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: operators.coreos.com/v1alpha1 2 | kind: ClusterServiceVersion 3 | metadata: 4 | annotations: 5 | capabilities: Basic Install 6 | categories: Networking, Developer Tools 7 | containerImage: quay.io/redhat-performance/cluster-impairment-operator:1.0.4 8 | createdAt: "2021-08-11 12:15:00" 9 | description: Run Impairments on Kubernetes/Openshift Nodes 10 | repository: https://github.com/redhat-performance/cluster-impairment-operator 11 | name: cluster-impairment-operator.v1.0.0 12 | namespace: placeholder 13 | spec: 14 | apiservicedefinitions: {} 15 | customresourcedefinitions: {} 16 | description: "cluster-impairment-operator is an operator designed to automate the 17 | application of impairments to the nodes in a cluster.\n\n## Features\n\nTraffic 18 | Direction:\n\n* Egress impairments affect the traffic going out. \n* Ingress 19 | impairments affect the traffic coming in. (Ingress impairments require the kernel 20 | module IFB.)\n\n### Impairments\n\n| Impairment | Description | 21 | Unit | Uses Netem | Correlation Supported |\n|---------------|--------------------------------------------|---------|------------|-----------------------|\n| 22 | Bandwidth | The bandwidth limit | kbit/s | Yes | 23 | No |\n| Latency | The delay of the packets | 24 | ms | Yes | Yes |\n| Packet Loss | The percent 25 | of packets that are dropped | percent | Yes | Yes |\n| 26 | Corruption | The percent of packets that are corrupted | percent | Yes | 27 | Yes |\n| Duplication | The percent of packets that are duplicated 28 | | percent | Yes | Yes |\n| Link Flapping | Turns the 29 | interface on and off | bool | No | No |\n\n\nOn 30 | the tested environment (RHEL CoreOS 48.84), the impairments can be used alongside 31 | link flapping.\n\n\n#### Latency Options\n\nIn addition to simply delaying the 32 | packets, there are advanced latency options.\n\n##### Jitter\n\nSemi-randomly 33 | adds or subtracts from the latency according to the distribution up to the amount 34 | specified.\nIf latency is 100ms, and jitter is 10ms, the actual latency will vary 35 | 100±10ms.\n\n\n##### Distribution\n\nThe distribution of the jitter. The options 36 | are:\n* Normal\n* Uniform\n* Pareto\n* Paretonormal\n\n##### Jitter correlation\n\nThe 37 | percent chance that the next latency's jitter value will correlate with the preceeding 38 | latency.\n\n##### Reorder\n\nThe percentage of packets that are not delayed, causing 39 | a reorder due to them being sent before the delayed ones.\nApplying jitter itself 40 | has the potential to also cause reordering.\n\n##### Reorder correlation\n\nThe 41 | percet chance that the value for the next reorder will correlate with the preceeding 42 | value.\n\n#### Loss options\n\n##### Correlation\n\nThe percent chance that the 43 | previous loss value for a packet correlates with the loss value for the next packet.\n\n#### 44 | Corruption options\n\n###### Corruption\n\nThe percent chance that each packet 45 | will be corrupted.\n\n##### Corruption correlation\n\nThe percet chance that the 46 | value for the next corruption will correlate with the preceeding value.\n\n#### 47 | Duplication options\n\n##### Duplication\n\nThe percent chance that each packet 48 | will be duplicated.\n\n##### Duplication correlation\n\nThe percet chance that 49 | the value for the next duplication will correlate with the preceeding value.\n\n\n\n## 50 | Configuration\n\nHere is an example of the ClusterImpairment custom resource.\n\n\n```\n\napiVersion: 51 | apps.redhat.com/v1alpha1\nkind: ClusterImpairment\nmetadata:\n name: test-impairment-cr\nspec:\n 52 | \ duration: 30 # seconds\n startDelay: 5 # seconds. It typically takes about 53 | 2-3 seconds for the Daemonset to run\n interfaces:\n - \"ens2f0\"\n ingress: 54 | # uses ifb\n bandwidth: 0 # kbit\n latency: 10 # ms\n latencyOptions:\n 55 | \ jitter: 5 # ms\n jitterCorrelation: 25 # percent\n distribution: 56 | normal\n reorder: 25 # percent of packets that will skip the delay\n reorderCorrelation: 57 | 25 # percent\n loss: 0 # percent\n lossOptions:\n correlation: 25 # 58 | percent\n corruption: 0.1 # percent\n corruptionOptions:\n correlation: 59 | 25 # percent\n duplication: 1 # percent\n duplicationOptions:\n correlation: 60 | 25 # percent\n egress:\n bandwidth: 0 # kbit\n latency: 100 # ms\n loss: 61 | 0 # percent\n linkFlapping:\n enable: false\n downTime: 3 # Seconds\n upTime: 62 | 3 # Seconds\n nodeSelector:\n key: \"node-role.kubernetes.io/worker\"\n value: 63 | \"\"\n \n```\n\n#### Interfaces\n\nThe interfaces configuration option is a 64 | list of all interfaces that the impairments should be applied to. The current 65 | implementation will error out once it hits an invalid interface.\n\nIf an invalid 66 | interface is found, it will print out the list of interfaces.\n\n#### Node Selector\n\nThe 67 | default node selector is all worker nodes, but you can change it to whatever node 68 | selector you want by setting the key and value.\nThere is a toleration to allow 69 | it to run on master nodes if the node selector includes master nodes.\n\n#### 70 | Duration\n\nThe number of seconds the impairments should run. It will try to sync 71 | the start and end time between all pods.\nIf set to -1, it will run indefinitely 72 | (a year), until you remove the ClusterImpairment custom resource. This is good 73 | for testing that requires steady impairments.\n\nIf the script is link flapping, 74 | set this value to be short enough so that if the link flap interrupts communication 75 | between the nodes, the daemonset will remove itself.\n\n#### Start Delay\n\nThe 76 | delay before starting the script. If you want the pods to be in sync, a minimum 77 | of a few seconds should be used because the pods take time to start up.\n\nYou 78 | can also utilize this feature to run an separate impairment after another. Just 79 | apply two resources at the same time, but add the duration and start delay of 80 | the first to the start delay of the second.\n\n#### Ingress and Egress\n\nThe 81 | configuration sections \"ingress\" and \"egress\" apply to each direction.\n\n##### 82 | Examples:\n\n##### Example 1\n\n\nIn this example, egress latency is set to 100ms, 83 | but the ingress latency is set to 10ms. So the latency to the interface will end 84 | up being 10ms, but 100ms going out. When pinging, this will result in 110ms of 85 | latency.\n\n```\n\napiVersion: apps.redhat.com/v1alpha1\nkind: ClusterImpairment\nmetadata:\n 86 | \ name: uneven-latency\nspec:\n duration: 60\n startDelay: 5\n interfaces:\n 87 | \ - \"ens2f0\"\n ingress:\n latency: 10 # ms\n egress:\n latency: 100 88 | # ms\n\n```\n\n##### Example 2\n\n\nIn this example, link flapping will be enabled, 89 | and it will turn the interface on and off every 2 minutes. That is long enough 90 | for kubernetes to determine that a node is unavailable.\n\n```\n\napiVersion: 91 | apps.redhat.com/v1alpha1\nkind: ClusterImpairment\nmetadata:\n name: two-min-flap\nspec:\n 92 | \ duration: 480\n startDelay: 5\n interfaces:\n - \"ens2f0\"\n linkFlapping:\n 93 | \ enable: true\n downTime: 120 # Seconds\n upTime: 120 # Seconds\n\n```\n\n##### 94 | Example 3\n\n\nIn this example, a realistic set of impairments is applied to `ens2f0` 95 | and `eno1` for 30 seconds:\n\n```\n\napiVersion: apps.redhat.com/v1alpha1\nkind: 96 | ClusterImpairment\nmetadata:\n name: typical-scenario\nspec:\n duration: 30 97 | # seconds\n startDelay: 5 # seconds\n interfaces:\n - \"ens2f0\"\n - \"eno1\"\n 98 | \ egress:\n latency: 50 # ms. Bidirectional, so total of 100ms\n ingress:\n 99 | \ latency: 50 # ms. Bidirectional, so total of 100ms\n loss: 0.02 # percent\n\n```\n\n##### 100 | Example 4\n\n\nIn this example, a combination of latency, packet loss, bandwidth, 101 | and link flapping is applied.\n\n```\n\napiVersion: apps.redhat.com/v1alpha1\nkind: 102 | ClusterImpairment\nmetadata:\n name: all-impairments\nspec:\n duration: 480 103 | # seconds\n startDelay: 5 # seconds\n interfaces:\n - \"ens2f0\"\n egress:\n 104 | \ latency: 50 # ms. Bidirectional, so total of 100ms\n loss: 0.02 # percent\n 105 | \ bandwidth: 1000 # 1000 kbit/s, about 1 mbit/s\n ingress:\n latency: 50 106 | # ms. Bidirectional, so total of 100ms\n loss: 0.02 # percent\n bandwidth: 107 | 1000 # 1000 kbit/s, about 1 mbit/s\n linkFlapping:\n enable: true\n downTime: 108 | 30 # Seconds\n upTime: 120 # Seconds\n\n```\n\n##### Example 5\n\n\nIn this 109 | example, packet duplicaiton is applied.\n\n\n```\n\napiVersion: apps.redhat.com/v1alpha1\nkind: 110 | ClusterImpairment\nmetadata:\n name: duplication\nspec:\n duration: 60\n startDelay: 111 | 5\n interfaces:\n - \"ens3\"\n ingress:\n duplication: 25 # percent\n egress:\n 112 | \ duplication: 25 # percent\n\n```\n\n## Setup\n\n### Requirements\n\n1. You 113 | need `make` installed.\n2. You need access to the kubernetes cluster with a kubeconfig.\n\n### 114 | Installation\n\nTo run using the current latest image:\n1. Clone the repository.\n2. 115 | Run `make deploy` with KUBECONFIG environment variable set to the location of 116 | the kubeconfig.\n\nTo run with your own code, there are more steps.\n1. Fork the 117 | repository.\n2. Clone to a machine that has access to the Kubernetes cluster and 118 | the kubeconfig.\n3. Modify the makefile to change the `IMG` variable to your image 119 | repository. If you do not have podman installed, also change podman to docker.\n4. 120 | Run `make docker-build` then `make docker-push`.\n5. Then run `make deploy`\n\n### 121 | Deploying from operator-hub\n\n*Important* You must manually apply the serviceaccount!\n\nFirst, 122 | Follow the instructions on this site to deploy from here.\n\nNext, apply the manifest 123 | \"serviceaccount.yaml\" from the manifests folder.\n\nThen you can apply your 124 | ClusterImpairment resources.\n\n## Running impairments\n\nFirst, configure a ClusterImpairment 125 | type's spec with the information for the impairment you want to run. Most importantly, 126 | set the interface(s). You can list the interfaces with `ifconfig`. If an invalid 127 | interface is set, the pod's logs will show `ifconfig` output.\n\n\nOnce the clusterimpairment 128 | type is set, apply it and it will work.\n\n\nTo re-run an impairment, delete the 129 | impairment with `oc delete -f path/to/manifest.yaml`, then apply it again.\n\n\n## 130 | Limitations\n\n\n### Multiple Impairments\n\n\nYou should avoid any configurations 131 | that apply impairments to the same interface on the same node. There are potential 132 | conflicts. The worker pod will attempt to remove all impairments on that interface 133 | before applying new ones.\n\n\nInstead, take advantage of the full control of 134 | both ingress and egress impairments from within the same ClusterImpairment resource.\n\n\nThere 135 | is currently a configured limit of 20 ClusterImpairment resources working at the 136 | same time. If that is a limitation for your use case, you can change it in `config/manager/manager.yaml` 137 | or you can try to find a way to make each ClusterImpairment do more.\n\n### Traffic 138 | Control (TC)\n\nTraffic control is how cluster-impairment-operator applies the 139 | latency, bandwidth, and packet loss impairments. The limitation is due to Linux 140 | not being realtime operating system, so the impairment will not be perfectly consistent.\n\n### 141 | Link Flapping\n\nWhen link flapping, if you flap the link that Kubernetes uses 142 | to communicate with the pods, you may be unable to remove the pod until the link 143 | is on long enough for Kubernetes to communicate with the impaired node.\n\n\nIn 144 | this case, it is helpful to set the duration properly instead of running for an 145 | indefinite or large amount of time, because the node will properly go back to 146 | the unimpaired state at that time.\n\n\nIf the cluster becomes offline due to 147 | the link flapping when you do not want it to be offline, soft restarting the nodes 148 | after removing the custom resource should remove all impairments.\n" 149 | displayName: cluster-impairment-operator 150 | icon: 151 | - base64data: iVBORw0KGgoAAAANSUhEUgAAASwAAAD5CAYAAACd+QhdAAABhGlDQ1BJQ0MgcHJvZmlsZQAAKJF9kT1Iw0AcxV9TxSIVB4OoOGSoThZERRy1CkWoEGqFVh1MLv2CJg1Jiouj4Fpw8GOx6uDirKuDqyAIfoC4uTkpukiJ/0sLLWI8OO7Hu3uPu3eAUCsx3e4YB3TDsZLxmJTOrEpdrwhBgIh+DCrMNudkOQHf8XWPAF/vojzL/9yfo0fL2gwISMSzzLQc4g3i6U3H5LxPLLKCohGfE49ZdEHiR66rDX7jnPdY4JmilUrOE4vEUr6N1TZmBUsnniKOaLpB+UK6wRrnLc56qcKa9+QvDGeNlWWu0xxGHItYggwJKiooogQHUVoNUmwkaT/m4x/y/DK5VHIVwcixgDJ0KJ4f/A9+d2vnJicaSeEY0Pniuh8jQNcuUK+67vex69ZPgOAzcGW0/OUaMPNJerWlRY6A3m3g4rqlqXvA5Q4w8GQqluJJQZpCLge8n9E3ZYC+W6B7rdFbcx+nD0CKukrcAAeHwGiestd93h1q7+3fM83+fgAtB3KLTzl0IgAAAAZiS0dEAP8AAADAqEO+QwAAAAlwSFlzAAAN1wAADdcBQiibeAAAAAd0SU1FB+UIAhUlMirtQycAACAASURBVHja7Z1pcFzXdef/b+290Qt2NABiX8WdorhJ1L5ZSrRYdpxUMpNUEmc+eGqS8ZRTmSlXMqmUZ5LKlD1LzXgqzsTjkVWSbEXWxkWSKXEBRRIESBD7vqO7ATQa6OXtbz6AoCiJILF2v9d9f1Uos0yKvO+++/73nHPPOZfSdV0HgUAgmACaTAGBQCCCRSAQCESwCAQCESwCgUAwOCyZAgJhG1FVaIIAXRShJ5PQFxehLSxAW1wERBFgGFBOJ2iPB3RODiiHAxTPg7LbQXEcmT8iWATC9qErCtTJSajT05Db2qBPTUH97DPop0+v/S+prATz/POgiovB7twJtqwMbHk5KLs96+eXImkNBMImRUqSoAwOQrpyBco770BrbQWGhrb2Q33ySTAPPQTu2DHwu3aBdjoBiiKCRSCsz+NRoWsaVE2DqiiQFQWqokBRVaiqCkWWIcsyFEWBpqpQNQ0rS46iKNA0DYZhwDAMOI4Dy7JgWRYMw4JlGTAsC45lQTMMaJoGTdOgjPCh6jrUYBBiSwvkn/8c2htvpO7frq0F953vwPLoo2ArK0HxPBEsAmH5u1xeHpqmQVVVCMkklpaWsBiNYikWQ2xpCbGlJcRjMSSTSQiCACGZhCgIUBQFuq5D07TP/64vL7eb4rMiXhRFgWYYWK1WWKxWWK1W2Gw22B0OuFwuOF0uOJ1OuNxuuN1u2O12MAwDmmFAUdT2i5muQx4agvjP/wz5pz8Frl9P6/uhv/1tWH/nd8AfOJAVwkUEi/DF7/GmtSQkk4jH44hGo4hEIpgLhzE9NYX5uTlIoghJkqDeFKSVn21frDdFjaZp8BYLeJ6Hw+lEYVERCoqK4PP74fF44HQ6YVsRMnqLDsJ1HcrkJJI/+xmU//7fgYkJ47w0lwv0iy/C+id/An7PnowWLiJYxIKCoihIJBKIzM9jZnoakxMTmJ+dRSQSQSIehygIpngWjuNgdziQ4/XCn5uLouJiFBYVwefzwelygef5DVlgWjQK4d13IX7ve8YSqjvA/Pmfw/YHfwCuqooIFiFz4k6JRAKRSARTExMYGx1FOBjEbDgMSRRTZjFtu7t0Mz7m9ftRUFSEQGkpSgIB5OblweFwgGXvcUiuaZDa25H8m7+B9otfmOfBd+4E/93vwv7SS6BsNiJYBPNZUbIsIx6LIRgMYnhoCMODg5ifnUUiHke2LAGKosBbLPD5/dhRWYkdlZUoLimB6w7WlxaLIfnaa5D+4i+AUMicgv1HfwTH978PtriYCBbB+CKlqiqi0SjGx8YwMjSE0eHhWzEowrILmeP1orS8HBWVldhRUQGP1wtMTSHxN38D7cc/Nv8z/t3fwfryy2BLSwHa/IUtRLAy0N2LxWKYnppCf18f+rq7EY1Ebp3YEe4My7Jwu1x42O1G1ZtvAufOZcaDuVxAXh4s//E/wvbSS6AsFiJYhPQjiiJCwSB6e3rQ39uLmakpYkmtA0bX8fjoKJpPnwYdi2WmKP/gB3D8q38F2uUigkVIPZqmYWlpCWMjI+jq7MRgfz9ii4vEklrvh6zreGZgAPUffACoamYL83e/C+d/+A+mFS0iWCZEURQsRCLo7enB9fZ2TE9MQJZlMjEbtKyeGRhAw7vvZs8z/+mfwvn974N2u823uZAlay6hCgaD6OroQGdHB2ZDIagZbhFsJzSAp4aG0PDee6mxDl54AXQgsOrv68EgtNdf3/ZxqH//94jxPJz//t+DdjiIhUXYWmRZRjAYxI3r13GttRWL0eitchfCBhc+gKPT03jg7bdBJZMp+Tft587BcuTIqr8vtbcjvmdPyuaA+9GP4Pj2t03VxoZYWAa3qMKhENpaW3Hj+nVE5ubIpGwRTQsLOPDRRykTKwD3ji2m2HaQv/MdJEtKYH/hBdN0fiCCZUBUVcXc3ByutbXh2tWrmJ+dJYH0LaRYEPDoxx+DDYezfi7EP/kTMCUlsBw8SASLsP4deHFxEZ0dHbjU0oLQzAwRqi3Gpml4/Pp1WEZHyWQAQCiE5Pe/D/b//B8whYVEsAhrQxAE9Pf14VJLC0YGB6EoCpmUbeDo+DgKzp8nE3H7RnnyJOI//CFcf/VXho9nEcEygPs3Mz2NlnPn0NnRASGFMZVsoyoWw66PPiITcad1+IMfQDh6FLZnnyWCRbgzS0tLaL96FZdaWjBr0gJbs8BrGo5fuwY6GiWTsQriX/81+L17wRQVGXaM5JqvNKAoCgYHBvDWG2/g1HvvEbFKAfcHg/C3tpKJuJtrePEikq++Chg4ZYZYWKlcELqOpaUlXL1yBRfOnsUS2e1TgltRsOfKlYwvu9kK5L/8S0hPPQW+qYlYWFkdI1BVjI6M4J/ffBMfnThBxCqFHJqYgG1ggEzE2uIUEH7yE+gGPfQhgpUCkokEWi9fxuuvvoqezk5yAphCPLKMhsuXyUSsZ3P9+7+H3NlJXMJsdAHn5+bw6ZkzaL9yBZIkkUlJMQcnJ8EbvA+7ERFeew1cUxMo1lgSQSysbXQBhwYH8YvXX8eVixeJWKUBh6qiqreXTMRG1u8PfgBlbIy4hNmAKIpoa23Fmz//OYYHBkihcpqoW1iAs6uLTMRG1/GpU0SwMp2lpSWc+fhjvPf221iIRMiEpAkKQNPwMJmITaD87/8NzWCHQ0Swtghd1zEbDuP9d97BuV//mmSspxm/JKGgvZ1MxGbW9NWrkDo6iGBlolhNjI/j7V/+EtdaW8kpoAG4LxTK2N7sqUT+8MOUt70hgrWNqKqK/r4+vPXGGxjo7SXdFQxCxdiYoT40067v99+HaqDQBklr2IyPryjo6erC+++8Q5rrGYhcSYJ7aopMxFZ4D5cvQx0dBePzEQvL1KayLKO9rQ3vvPUWESuDEYjFYBkfJxOxVWvdQLFAIlgbQJIkXLl0CSfeeQeLpMTGcJTPzoI4g1voSZw4YZhSHSJYGxSr0x98gDgJ6hoORtfhWVggE7GFaGfOQI/HiWCZUawuXbyIU++/j2QiQSbEgNg0Db6hITIRW0koBMUgOW1EsNbqx9+0rD46eRKiIJAJMShORQE3PU0mYotRDdKzjQjWWnx4RUHrlSv48MQJkhBqcAoN4rpknFs4MkIEyyxida2tDR+eOEHcQBNQsLhIJmE7voNLl4hgGX5X0TT0dHfj1PvvkwC7SeBJlcH2IIqG6NhKBGsVdF3H0OAgTrz7LkldMBFWI8cX79FiSDdwCyI9HjfE+Eim+ypiNTU5iRPvvos5cjvw+nZAmgbDMKBoGhzHged5cDwPhmFAMwxoigJF04Cu3ypjUhTl1o8sy5AlCZqqQlXVdbfmsRvYbb9XaoBu5PhoJAJdEEDZbESwjMb8/DxOffABpkinynsKE8fz8Pr9yMvLg9vjgdvthsvtht1uB8/zsFgsYDkO7Ipg0TQoirolVrquQ1XVZcGSZciyDEEQkEgkkIjHEY1GsTA/j7nZWczPzUESRaiqCv02wbsd29ISEaztGHsoBN0A1isRrC8Ri8Xw69On0d/TQwqZb4OiKFhtNvj8fuQVFKCgoAD5hYXw+/1wOBzgeB4sy4KiqFs/m7VygeU44orllUwmMRcOY2ZmBuFQCKFgEHPhMERBgKZp0AHwBo416vcYm27kQ53xccyOj4O3WOBNY10hEawvhBgkfNbSgmtXrxKxAsBxHHK8XuQXFqKyqgolgQA8Hg9sN62nzYrSvQQSABiGAcMwsFgscDqdyMvLQ11Dwy0BW4hEMDM9jaHBQUxNTIA28HvT72H96UZOyVhawof/68cQH3gA3/jWb8HhcBDBSieqqqLrxg1c+PTTrO5nxbIsvH4/KqurUVldjcKiIng8HnAsuxx7Moi1x/M8eJ5HTk4OysrLsXf/fixEImA+/RQwaKdR/R4pF7qRT6IpCrFwGG0tLaioqsSxBx8Em4YLKohg4fMGfB+ePIlEFiYe0jQNp8uFHZWVaGhqQmlZGdw5ObdcPDO4qxzHIS8/H4vV1TDqdal6PL7co2uVOdUNHH/TeB4iw0AURZz+4AQqKiqwo6KCCFY6WFhYwMenT2fdiSDLsigqKUHTzp2orKpCXn4+LBaLKURqVfEqLjbu4JJJ6Kq66tVZuoGTXjWrFTGGAQBMT03h1MmT+Nbv/A6cTicRrFQiiiJazp3DQBZdB8VbLKioqkLTzp2ora2F0+UCc3Mxmt5aLCw0riUfiwGKAtxJsFQVMHAOmcpbsHjbGrny2SU0Njbh8NEjoFMYKshqwdI0Dd1dXbh88WJWXMVlsVpRWV2N/QcPoqy8HA6Hw9TW1B0Fy+s17uDi8WUL605ipmmGTmvQOPYLgiVLEj46fQrVNctxTiJYKWB6ehq/Pn064wuaeZ5HVW0t9h44gIrKStjt9owTqlukObHxrhbWwsLq5S2KAhg4rUGjGSQoCjRFgWGWc/BCMzO4cvkynnz6aXAcRwRreze7OM598gnCwWDGPiPDMCgsKcEDhw+joakps4Vq5ZmNHMOam1tVsHRVNXQMK5qXC47nwTH0F1zAi+fPo76hAVXV1SlZW1kpWKqqouP6dXRdv56R+VYURcGdk4ODR45g5+7d8Hq9KY0zEMFaxcK6enX1VsOqChi4ZnU2Nxc8x37FnV2IRPDpJ5+guKQEdrudCNa2uIJTUzh35gwkAxebbhSO41Df3Iwjx46hJBBIS65MOqFtNlB790K/etWYorXKmtNVFfrsrHEtLJsNq9lPXdevo2f3buzeu3fbN8asE6xEIoEL585h3sCLY6Pk5ufj6PHjaL7vvrRlIqfduuR5MC++CMWggiV+/DGUO5xkatGocRNeGQb9OTmr/r4gCLhw/jwqq6vh8XiIYG0Vmqahu7MTndeuZZQryLIs6pqa8ODx4ygJBDImRWFjPiEDpqICRq1VkH73d003pYrPhxmev+ufGejrw43r13HoyJFtXX9ZJVhzs7NoOX8+o1xBp8uFww8+iP333w+XywUCwO7eDZFMw5YxU10N8R6uniLLOPfpp6hraEBeXt72ufzZMumyLOPK5cuYypALNimKQlFJCX7j5Zdx9MEHiVjdbmQVFoI6fpxMxBYx6fNhLVmKUxMT6Lh2Deo2dibNGsEaGx3F1cuXM8IVpGka9U1NePEb30BjU1PKcmBMI1g+H+iDB8lEbAE6z6Nvjcm4mqbh0mefYW4bb0LPCsFKxOO41NKCuIGLS9cKx3HY/8ADeO6FFxAIBLImXWG98I8/TiZhC4hWV2PMal3zn58aH8f1bbSyMj6Gpes6Bvr70dPVZXrrymK14uhDD+HQ0aNZewq4ZmHftQvJQAAgXWM3xWggAIGisJaUUJfbjfKKCng8Hmiati3B94wXrKXFRVy8cAGSaO4wrN3hwKNPPol9Bw7AYrGQL+lebqHfD/YP/xDK979PJmODaA4HruXlrSpWFEXBYrGgsLgYTffdh7r6ehQUFsJms22b5Z/RgqVpGrq6ujBukEsgN4rT5cITzzyD3Xv3knjVWqEo8E8/TQRrE8zV1GDgS9nrFEWBZhjkFxSgoakJdfX1CJSWwulwgElBknJGC1Z0YQFXL10ydQdRV04Onnr2WezctQssEat1wTc1QfjmN6G99pohxkP/0R+BrqxcPXwxOQn1v/5Xw8xfe2Ul9NuEyuf3o7q2FnUNDaisrESOx5PyDTRjBUvTNHR1dpr65huny4Unn30WO3fvzroSmy0xsux2cF//OkSDCJb1d38XliNHVv19qa0NcYMIVryqCu1eL5wuFwJlZdi5ezcqq6qQm5ub1iaPGfsVRObn0dbauq05IduJ3eHA408/jV27dhGx2gSW48chHTgA/fLltI/FTIc+k7/5mzj28MOorqlBcUnJtsalsl6wNE1DX28vpk1qXfEWCx5+7DHs2buXuIGbhPH5wP/pn0L8rd8ik7FWy/TrX0fT974Hm8cLhjVWmVdGJvEsRqNovXzZlF1EWZbF4WPHsP/gQXD3qN8irNEVe/ppkvm+Dmz/+l/DmZtrOLHKSMHSdR19vb0ITU+bbuw0TWPXvn04cuwYrOtI1iPcY15zcmD9i78gE7EWi/TP/gy8gasEMk6w4rEYOq5fhyzLpht7ZU0NHn7sMThJXeCWYzl6FPS3v00m4m7k58P27W+veqsPEaxtsK7Gx8ZMmXeVV1CAx558Er40XgOeyVBWK+z/5t8A+flkMlaB/8//GVxVlbGt5UyacFEU0d7WBtHA1yXdiZUs9tKysozvuZ5OuJoaWH70IzIRdxKC3/s92F56adVLXolgbQPBYBCDfX3mWig0jQOHDqGhsZEUMm+7mUXB9txzYL7zHTIXt9PYCPv3vgc6xZeiZrVgqaqKvp4e0101X1VbiwcOHwZPTgRTo1l2O+z/9t+C2rs3DXppTOvF+p/+E7i6OlO8v4zJw1qMRtFrso4MHq8XDz3yCHLu0i+bsA2LvrQUtv/5P5H45jeBoaHUhSxefRXyxYur/r6WhrxB7oc/hPWppwzvCmaUYOm6jtGREQRNlMrAMAweOHoU5Tt2kLhVGrDs3w/thz+E8K1vASnqk6b+j/8BI9VdMN/9Lux/8AeGPhXMSJdQEkV0d3WZqgynoqYGu/fuJWU3afMNKVifegr83/5tVj4+/corcPy7fwfaZH3VMkKw5ubmMD4yYhp30OF04uiDD8LtdhPhSKdmsSzsv//74H74w+x67hdegOO//BcwubnmE1qzT76maRgZHsZCJGKSjZ3C3gMHUFlZSVxBI7wPjoPjj/8YXJakO9AvvQTnf/tvYA18Q3ZGC5YkSeju7DSNdZWbn489+/eTOkEjiZbFAscf/zH4H/84oxNL6d//fVOLVUYIVigYxIxJgu0Mw2Df/fcjn2RbG0+0eB72f/kvYf3HfwRqazPu+di//mu4/u7vwNzh1mkiWCl0B0dHRpA0Se5VcWkpdu3end03MxtZtFgWtqefhuPnPwd1l0Z7piI/H/xPfgLnn/0Z6DVe10UEa5sQBQEjQ0OmOB1kWRb7778fbpJzZXDVosDv3QvXG2+A+e53zf0ox4/D/vbbcPze74HKkO4fphasxcVF07RALgoEUFNXR8pvTAJTVATXX/4lrG+/bUoXUfjDP4T9Zz+D5YEHgAxac6Z9El3XMTk5iZgJLkdlGAb7DhwgGe1ms1BsNtieew6uDz8E8+d/Dpig7Y+0Yweu/vZv49SuXRBMUBuYNYIlyzKGBwdNcSNOYUkJsa5M7CKypaVw/dVfwX7qFOh/8S8MOUw1Px/DX/safvnCCzhfVoapUAjTU1Omvzz4y5g2zTqZSGDMBH2vaJpGQ1MTsa7MrlssC8sDD0Cvr0f4scfA/ehHsFy5AqS5DbfmcmH0oYdwtaoK03b7rWu5FEXBQH8/qmpqMqqawrRPEg6HEV1YMPw43Tk5aGxuJieDGYCmabjR14dzk5OQHn4YzU1NqOvtRU5vL5i5uVTugkg0NGCksRHXSksxZ7FAv0PIZHx0FEuLi/BmUFNIUwqWruuYnpoyxfXz9U1NyDVhCQThztZyQWEhdF2HQNO4UliI1sJClO7bh+pgEMVjY/Bevgxs06l1sr4eoaoqDAQCGMrJgXCPTTAyO4tQMAiP15sxVRWmFCxZljE5MWF4/9xitaKuoYFcL59BFBYVobG5GZcuXICu69ABjDmdGHM6wVVUIOfwYVTPziJ3fh6FAwPgolEw0SioRGLt/wjDQPX5oNpsmK+uxlxuLgYKChC22ZBch6UuyzJGhodRXVubMRa+KQUrHo8jHAyaYnEXm7gMgvBVOI7Dfbt3o7uzE4tfCknINI1ZiwWzJSVASQmo++6DQ1FQHIvBKYqwShIcggB7IgFrIgFGUaDRNCSrFQm7HQmbDUmLBQLHYdrpRMRigbpJy2h4cBCJRAKuDLnYxJSCtRiNYm521tBjpCgKNXV1cGTg0XK2k19QgIbmZlw6f/6uVr4OIMay6PN4vvD/rbpmtmGs0UgEwZmZjBEs052z67qOmelpyJJk6HHa7HZUVleTYHsGwrIsGpqaNnQdG3WXn20Jn0gSpicnoZmoV1xGCZamaabILyksLkaByQtNCXd5v4WF2GHwK7FWvpfJiQlIJrynMyMESxQERCIRwwtWdW0tub05g+F4Hs07d4K3WAw/1nAwiHgsRgQrHSSTSURSmfOyAaw2GwKlpSSzPYOhKApFJSUoDgQMP9Z4LIY5g38zGStYi4uLht8tPF4v/CT3KuNxOByobWgw/MYkiSJmpqagpTkrP+sES9d1LC4uIrmenJY0UFRSkjGnMoS7W1mVVdVwmqA3/+T4BCSDH1RlpGAtGDx+RdM0dlRUkNtwsgSP14MKEwTfQzPTGRHHMpVgaZqGcDBk6DHyFgvyCwrIBRNZAsdxywXGBq9mSMTjmJ+fJ4KVSlRVRShk7Ax3n9+PnNsSBQmZ7xYWFhcb/p2rqoq52VnTt5sxlWAJyaTh41e+3FzYbDbyJWcRbrfb8KeFmqYhHAqZ6rJh0wtWPB6HaOAODRRFwZ+bS4qds9AtLK+oAG3wqobI3JwpOpxkjGDFYjFDTzhN0/D7/aQcJwvdwpJAAA6DX/seXVhAMpkkgpUqkskkVAO3RGYYZsP5V5qmQRQEU7R8JnyVHI8HeQUFxt7wFxcRM/lJoWkES9d1JBMJQwcNOZ5f1zVeuq5DFEVMTU6i5cIF/OM//ARjo6Pk6zchPMchUFZm6DGqmobI/LypA++mSRbSNA2JeNzQk+3OyYF1DbVliqJgMRrF8PAwrrW3Y6C3D6FgEFabDU8+/RT5+s3oFt7sRspxHGSDFhrrmnbrpNCsaTemESxd15EwuIXl8/tXzcfRNA1LS0uYnJxEV2cnum7cwMTY+Bfa5PAWHgxJODWnYFEUvD4f7A6HYe8a0HUdi9EoVFXdknIiSZIgy3JKY3emEixBEAwtWE6XC+xtAfcVkQ2Hw+jt7sa1tnaMDA+vmprBcfwX/nuCuXC5XHB5PIa+HCUej0OW5Q2fZCuKsrzxTkzgxvXr8Ph8eOLJJ1NWT2kql9DoTfscDgcomoYkSZibnUVvby+6bnSir6cHsaWle+bAsCxDOjyYGKvNBr/fjwkDXz8XW1qCJEmw2+3r+vYEQcD42Bh6e3rQ092NyfFxSKKI4489llIjwjyCpaqGLt6kaRqiJOFa+zV0dnTgWns7opHIuk79dE0nX72JoWkahUVFuE5RhvUExGRyTalBmqZBEkWEw2EMDw2h7epVjAwNQUgmv/BssiwTwbrjBOq64Y78KYoCRVHQAQiihA9PnoKQTG44uVXX9Yy7qTeboCgKXr8fFE1DN2hGuSzLd83FkmUZkfl5DA4OorOjA8ODg5ifm1t1XSYTCSiynLJif1O5hKqiGmZhgqIhSSIEUYQoiFsipjoAolfmxul0wm63I7a0ZMjxSZJ0Kxa8clKoqiqWFhcxPj6Ors5O9HR1ITgzs6Y+8JIkQU1hny1TBd1VLT2CdfsRsKJpSCYSEAVxS46vV6w0hqFhtVrAsCTobmZsdjvsDodhBUuRZYiiCE3TkEwmEQoG0dXZiRvXr2NqYv09sxRFSekFF+YSrDSY2QzDQFZUiKIAURQhitKWuG00TYFhGDA0A5qhl0VR1zPmdpNsxWKxwLaOgHY6Nt7hwUGEQyH0dHdjeGgI8aWlDa9pRVGgkRjWHRUrJf7SisWj6TpkWcFCNApJlLbkpVAUBZqml4WKYUDT1FfMa5mU5pgalmE2dP3XdouUDkBWVYiShI9OnYKiqluyOWqqSoLu6Xipy8FzajkuJYgQRXHLgvwM87lIrfxbd0JIJg2fukG4t0VudziW11MaA5LLcVZquUIkKUAUBUiybHoLPqsFayXnSZIViJK0fOS7JXEpgKLoW0JF0/SaSiEURYEgCOSrN/OaYpi09UOjKAr6TaEUJQnJZBKiJBm6YUDmCtbNHWMrd59EUkAikdiyXBKa/qLLt5F6raWb8QTSYtm81rrFYkmZhbWylnVdhyTLSAoCkoKQUSJlSsFaPknb2Akay7Jwud0oKS1FX28vwsEQREmEpulbNK7PLanNZKrruo7owgIRLJOTqstVaYZZTlOQJIiCAEmSUu6GUjSNVK7UjBUsmqbhcrtRVlGB6poaVFZVgWYYzASDGBsd2/RYaIoCw947LrVewZoNh6FrGkBKdEwtWBRNA1ucn0QBoBgGiqpClqTlDrxpEKnb4Vh2+VmJYH1VgNYiWDa7HSWlpairr0dldTW8Ph+sVisoioIoCJu6LODLJ3zbYQUtLCxAkmXYSNcG08Kx7JatjdtdPkGSkFxcNFRcimHZlNa/mkewKOqO6f8URYHjOOQXFqK6rg7VNTUoKCxcLkT+0qLheB5ud8664gu3u3wbjUuth6WlJSQSCXKRhYnZbG93iqLgcDpRFAhgbnYWI8PDEEXRkH22eJ5PaUtw1kyLgOf5zwfOsvD4fNhRWYm6hgaUlpbC6XLdc/IsVsuaFszn1hSd0h1kcWEBsVgMfr+ffPlmFawNbmo2ux35hYWoqKpCZXU1fH4/3vrFL3Cjo8OwYRqb3Z7SS4NN5RLyFgucLhd2VFahobkJgUAAHq8XHMetyfKhKApejwc0TX8la34lFYHd4rjUel4+x3Hw+nykoDBLoCgKDMvC4/WitqEBFZWVKCgshN1uB80wUBXFkHl5NE2DY1nwPA+3y0VcwtUmadeePTh46BDy8vPB8/y6BYWiKBQUFYFlWaiq+nm+FE2DYdeeL7XVMQB/bi5qamtR39CAiqoquAyUKU1YP/eqimBYFu6cHJSWl6OqpgZl5eVwOJ1fsVREUcSSQWoSKYoCy7DgOBYsy4KhGVA0BbvdntJvxjSCxTAMqmtqNv33OJ1OWKwWqKoChmXBpEGkKIqCy+1GZXU1GpubUVVVBY/XC0uKjsMJ28udal5X3KeSsjLU1tejrLwcOTk5d02BSCST2O0RawAAGWtJREFUae1eunIyz7IsOJb9SgyXoqjlrP4UjinrjqIcDgdKy8sxMjiY8pdvtVpRWl6OxvvuQ01tLfLz82+dYBIyB+VmIjJFUeAtFuQXFqKqpgYVVVXIzctb8zuXRBGJeDy16xTL8eIVkWLvcuK5cjgAYmFtHzabDT6/P2WCZbFYUFBUhOraWjQ2Nd26cJO0Qs5gwVIU5OXno2zHDlTX1aGwqGg5LrXOdx6LxVJWqkXTNFiGAcdxYNeYqkBRxCXc/gdmWfj9/m0tnWBYFm63G43NzWhsbkagtBQ5OTkpPU0hpI8dlZUor6iA2+3e8DvXdR1zc3PbKlhfjkutN4abjrrJrPuCaJpGfn7+lgvWSmb9jqoq1NfXo6a2Fj6/f80nmITMwev1bvrv0HUdC5HIlndXoCgKDE2D5bg7xqXWg9PlgtVqJYK1rT46RcGfmwue57dk97LZbAiUlaGxuRm19fXIzc2FzWYjLh9hU8iyvKW3gDM0DZZlwXPLiZ4UvflN1OF0pvygKCt9FK/XC4fTuWHBYjkOhcXFqG9oQF19PYpLSuBKcT4KIbMRBAGhYHDTVv9641LrwW63p6zQO6sFy+5wwJebi7nZ2XW9/Nz8fFRUVaGpuRk7duyAm8SlCNtEKBhEZH5+/SJFUctVIbeJ1HaFJHI8npRf/JuVX5vFYkFZeTn6e3ruKVJ2hwNVNTXLIlVZCb/fv6GkVQJhrei6jpmZmVVvCL9TmIOmaXBbEJdaKxRFwevzbbpukgjWWh6aZVESCIBh2a9Uva/kSwXKy1FXX4+6+noUFRWBt1iIy0dICbIsY3ho6J6HQitxKY7jbvVjSxUURcHj9ab8m8haf2alo8NiNLo8ERyHgsJC1Dc2oqGxEUXFxXA6nSmtRCcQgOX8q8nx8VVcvuUyMp7jblVqpAPeYkFOTk7qjY1sXRSenBz4c3MhSRICpaU49tBDKK+ogMfjIXEpQlqZm51FOBT6gjXDMsytVIR01Lx+GZvdDjcRrNSZ3IIgID8/H7IgIC83F41NTYa9T46QPWiahpHhYQjJJBiGAXeby2ekuKn/ZvoOEaztWgiqingigcnxcQwPDmJ4aAgL8/NQZBmR+XnE43EiWIS0I4oiRoaHYbfZUh6XWg8FhYVf6E9HBGsL0HUdyWQSs+EwhgYH0dvZicj8POQv9cGOx2KIRqPIzcsjXwwhrUQXFhCangbHcYYdI8MwyC8oSEt8N+MES9d1yJKESCSCkeFhDPT1YXpi4q5V75IoIhQMorKqiqQrENK6doeHhtaczpAueIsF+QUFaflWMkawZFnG0uIiJsbHMdDXh/HRUSxFo9DWcHOJruuYnpyEoiiG3tkImU0ymcRAf/8d+2kZiRyPZ1OXuWStYKmqikQigeDMDPp7ejA0MICFSGRDN4rMTE0hHo/Dk6YXQSDMhsMYGx42/Dhz8/PTdkmK6QRL13WIoojgzAyGh4Yw1N+PcDAIcZOFzIsLCwjOzBDBIqRt8+3t7kY8FjP0OGmaRlFxcdo8EVMI1u1xqfGxMXR3diI4NQUhmVyTy7cWJEnC+OgoqmtqSLIoIeVEo1H0dnen9VLUNQkGx6GouDht34ihBUtRlFtxqf7eXkyMjSEaiWzbSx0bGUEikSCXQBBSviEPDw4iOD1t+LG63G4UFBamTzCNNiEr+VLB6WkMDgxgeHAQc+EwNFXd9t1nLhzGbDhMBIuQUpLJJK63t0MxyG3Od6M4EEjr92EIwdJ1HYIgYG52FkMDA+jv7UU4GIQkiikdh5BMYmx0FGXl5cQtJKRs7Y8MD2PUBMF2iqJQVVOT1tI1Np0vSlEUzM/NYWx0FIN9fZgcH0ciHk+bH6/rOvq7e7Bv/344iZVFSJF11dbauulDo1TgdLtRXFyc1lzFlAuWqihYXFzE1NQUeru6MD4yglgstqFUhO0gHJzBxMQE6hsayNdE2PYNcmWzNgP5BQXLN5OnkZQK1tzsLK5cuoTRoSGEQyHDiNTtKIqC/t5eVFZVpaVWipA9CEkBbVeuGD6zfcUdLN+xI235VyuktLJSEAR0tLVhZmrKkGK1susNDwxsqD0tgbCedTY40I+eri5TjNditaLKACk/KRUsd04OfLm5hn85iwsLGOjr27IcLwLhyywtLeGzixdTfrC0UYpLSpCXn5/2caRUsKxWK4rSHLRbC5qmoevGjVvdSAmErV5f3Z2dGBkYMMV4KYpCRVUV7AZov5RSweI4DsU3e6kbndDN0h+jZx4TzEc4HMbF8+dNkXcFAFabDZXV1YZI9Ul5d7DikhLYHQ7DvyRVUXC9rQ3xu7SlIRDWiyRJuHzxoimy2lcIlJWhMI3Z7WkVLI/Xi+KSElO8qMmbrWqIlUXYCnRdx0B/P9quXDHNmmIYBg0Gah+ecsFiWRZVtbWmaJSnyDKuXb2KpaUl8rURNk1kfh5nP/nkrs0kjYbb40GFgRpbplywKIpCUXExXGm4cWPDVlZvLzkxJGzaFfzs4kWMDQ2ZZswURaG6thZer9cwY0pLh3uvz4fCoiJTvDRFlnH18mVEyYkhYROuYE9XFy63tJhq4+MtFtQ3NsJisWS3YFmtVlRUV5vmJuWZqSl0mKSanmDA9TM9jU8+/tgUGe23U1RSgvIdOww1prQoBkVR2FFRAZfbbYoXp2ka2ltbEZyZIV8fYV0sRqP4+MMPMT05aapx0zSNPfv2wZ7mUhxDCNaKW1hqMPW+G9FIBJdaWpBMJslXSFgTkiThYksLujs6THfSXFBUhKrqalAG84LSNhqO41BTV2eKJNKVOERvZyf6enpIAJ5wTxRFwbX2drScPWv4W3DuZF013ncfPAYKtqddsCiKQmlZGfwmqC28fcdsOXcOc7Oz5Isk3DWE0N/bi49OnIBgQos8x+tFU3OzIZtYptXec7ndqG1oMNXlpeGZGbScO0dcQ8Kqlvj42BhOvv8+ogsLphs/RVFo3rnTsLegp1WwGIZBTV0dHE6nqRZkV0cHum/cMJ2pT9j+tTE9NYUT776LkEkPaJxuN+7btcuwFwqnPaKWl5+PsooKU71USRRx7pNPMDk+vu2uhSzLpDTIJGIVCgbx3q9+hdHhYVO+M4qisHP3bsPUDRpSsCwWC+oaG03X3TMaieDTM2ewEIls+cKXJAnj4+M4d/Ys3v7lLzE9PU1Ey+ihgnAY77/zLoYHBkz7rjw+H3bv3QvOwN9i2o/oKIpCRWUl8ouKMDE6aqoddWRgABfOncPDjz226daxsixjYWEB42NjuNbWhoG+PkQXFqAoCiYnJ/H1b3zDFL3EstGyCs7M4L133sFgb69pxYqmaTTv3JnWOwdNIVgA4HA4sGvvXkxNTEAzUVxI0zRcu3IFXp8PBw4eXPf1R6qqIh6PY2J8HF03bqCnuxszU1NfyajvvnEDr+s6Xnz5ZQRKS01TIZANYjU1NYX3f/UrU1tWK9bVnn37DBu7MpRgrXQ0LCgqwvTEhKletCzLOH/mDJxOJxrXcBSsaxqSgoCZ6Wn09faiq7MTo8PDkETxrgu+p7MTrwoCXnz5ZVTX1BDRMsBmNToyghPvvYfxkRFTixVN09h/8CDyCwoMP1bDZG16PB407dyJ4PS0qawsAEjE4zjz4YewOxyovEMrjpW41PzcHPr7+tFx/RpGh4extLi4roU+MjiIn/3TP+Hr3/gmGpoa03qhZTajKAr6enpw4r33EA4GTf88xYEAdu7aZYrLgyndQFvD/Nwc3nj1VVN1Y7ydwuJiPP388wiUloKiKKiKgvlIBGMjI+ju6kJvdw9mw6FN78Zenw9PP/cc7r//flgNVuuV6YiCgPb2dnx88mRG9PznOA7Pv/QS9u7fbwqr3VCCpakqWs6fx69PnTJt+UugvBwPP/44RElCZ0cHum7cQDgYhCRJW/rvWG02HH/0URx/5BF4PB6iJCkgGo3iwrlz+Oz8eVPc1LwWGpqb8eIrr8BpklxIQwkWAMyGw3j9//0/zIZCpnnpt1xAmoYoSmBYBtFodNvLMmiGwa49e/DUM8+itIwE47dtI9U0TE9P4+NTp9DT2ZkxtaROlwuv/PZvo6a21jzfmtEES1VVXGppwUcnTxo6lkVRFEBR0HQdqqJAEEUkk0nIspzysZQESvHU157FfTt3wmq1EoXZQgRBQHdXFz756COEZmYyJh+Opmkce/hhPPrEE4Y/GTS0YAHLva/ffO01Q54YrgiVoihI3hQpxQDZ6Da7HYePHcNDx48jNy+PWFubRNd1zM/N4fzZs7h6+XLGuIC3hy6+8a1vGbZmcDUMecyU4/Fg9759CAeDUNJgsawqUqoKSRQhCAJEWYKmGMcCTCYS+PjUKYwOD+PRJ55AQ2MjsbY2iCgI6O3pwflPz2JibDTj2glZbTYcffBB+Px+043dkBYWAMRiMbz1+usYTtPtuLe7fLIsIxaPQ5ZlqAZuk0xRFGiaRk5ODg4dPYoHDh9Gbl4eyY5fRzgiGAziswsXcL2tzZStYdayRg4cOoRnnnvOUL3aTW1hAcvZ7/cfOoTpycmULZwVkQJFQZZlxBMJSJIEeYtP+LYjHsGyLHiOA8Mw0DUNLWfPYnRkBAcPHUJDYyMcTicRrru4f9FoFNfb23Hls88wGwplbO1mcSCAw8eOmVKsDG1hrZjmJ99/H9daW7d1AVG3iZQoyxCSSUiSZOhFS9M0GIYBx3FgWRbMKjErjuNQWVODg4cOYUdl5aZrHjNNqOKxGAb6+/FZSwvGR0YyumWQzW7Hb7z8Mnbu2mXazcvQggUAU5OT+MVrryEyN7elfy/DssjJyQFvtWJ0ZATxeBySJBk6XkFR1LJIsRw4jgVN02teeFabDXUNDdh34ABKy8thsViy1uLSdR2JRAKD/f24cunSrdKoTIamaRx56CE8+sQTprWuTCFYqqri4oULy8mkm9z9aJqGw+lEeWUlKqurESgrgyzLePWnP0V/b68xXxCW8604lgV30+XbjNDYHQ7U1tdj5549KC0rg8PhyBrh0jQNS0tLGB4cRPvVqxgeHMx4oVqhqrYWL73yCrw+n6mfw/CCBQCLi4t4+803NxSApygKNrsdhcXFqKypQWV1NbxeL3ieB0VRy5dcdnfjpz/5CSLz84Z5ZoamwdwUKZZhtjxNwWK1omzHDuzauxcVlZXIyckxRS3ZRpBlGZFIBAN9fbje1obpycktrzwwMl6/Hy9/85uoqKw0/eZkCsHSdR1Dg4N4+403EFtaWtN/w1ss8OXmorq2FlU1NcjNy4Pdbr/jC1MUBZ+eOYO33ngjLYmftyxAigLDsOD4ZZFiaGbZxNpOYWQYFJWUoKa+HnX19cgvKIDFYjF9HpeqqhAEAZMTE+jp6sJAXx/mwuGsu/GIt1jwzPPPY//992fEhmQKwVrZJc+eOYMLn366qmvIsCw8Ph923HT5SkpK4HS51vTxxWNxvP3WL3H2zJmUBts/j0uxYFkODEOnZRekKApWmw0V1dWoratDoKwMPp/PVOK1IlKhYBCjIyMY6OvD+OgoZIMfoGzbBkjTOHTsGB578smMyckzjWABwMLCAv75jTcwNjz8hZficDpRUlaGuoYGlJaVweV2g2XZdX/4s7OzePWnP0XXjRspcflYlgPPbz4utR1Wl8PpXLa86upQelO8rDabocaq6zoURUEiHkcwGMTY6CiGBwYQnJlBMpHI6vsjKYpCXWMTfuPFFwx5v2BWCNaKa/irX/wCkiiiuLQU1bW1KN+xAz6/f9MnXyt//z/9wz8gtA19jlbypZatKdYc7TwoCg6nc/mm7vJyFAcCyMvLg8vthtPpvCVg2y1iuq5D13XIkoRYLIZIJIJQMIjpqSlMjI0hMj+fceUzm6E4EMCLr7yCkkAgs4RYN5mtvNI8zW63I7+gAFabbUs/fE3TcK29HT//v/93S/od0SsuH8eB5bhV86XMAsuysFityPF44PP7UVRcDH9eHtxuNxwOB6xWK9iV3LCbhwVrFTNN06BpGlRVhaooECUJyWQSyUQCkfl5zMzMIBwMYiESwWI0CkkUyS3cdyDH48Fvfv3rqKuvz7gTYNMJViqQZRnnz57FW2++uaFde6VEhr/tw83U1IEV64rlOLjcbtjsdtjtdjhdLjgcDnA8D57nYbFYlk9maXo5VYOmoaoqZEWBIsuQJAmiKEJIJpFMJhFbWkIiHr/1o6rqLSuLsDo2ux3PPP889uzbl5GnvkSwViGZTOLUiRM4/cEHX7kUYlWRoiiwN5M6NxJDyzQhu5OIr+bq3b4MyZLcGDzP4/jjj+PIsWOmuzZvzRY+ec2r7FQ2Gx557DHEYjGcv8vJ5OdxKQ4sy5C2LquIjq7rxH3bRhiGwf2HD+OBw4czVqyIYN0Dl8uFZ559FoIg4MrFi7c+QoqiwDLsLUtqPXEaAmGroWkaew4cwIMPP5zxtaJEsO6B1+fDc88/D1EQ0NXRceuUL5PjUgRzud7Nu3fj0ccfh8vlyvznJTGstbk3oVAI7//qHfT3dJMYC8EwYtXQ3Ixnnn8efhM249uQNUle+9oWRkFBAZ55/jnUZOBRMcGca7K+qQlPfe1rWSNWRLDWSX5+Pp557jlU1xHRIqRXrOoam/D0176GPJP1ZCcuYZrcww/eeQd93cQ9JKTYwqBp1Dc346lnn806sSKCtQnRCofDOPnee+i+cYOIFiFlYtW8ezeeePrprHIDiWBtEfPz8zj9wQfoaG/P6Na6hPTDMAz2HDiAR594Iqtv+iaCtUmi0Sh+/eGHaP3sszVlxBMI64XneRw8cgQPHj8OZxakLhDB2mbi8Thazp3DuU8+IR0DCFuKzW7Hg488goOHDpELRIhgbR2iIOBqays+PnVqzV1RCYS74fX78egTT+C+XbsyutyGCFaaUGQZPd3d+PDkSQSnp8mEEDZMSWkpnnjmGVRVV2dsr30iWAZA0zRMjI/j9IkTGOrvJwW/hHXBMAxqGxrw2JNPoqi4mOT7EcHafnRdx/zcHD49cwbtV65k1Q0thI1jsVpx/6FDOHLsGHKy+CSQCFaaSCQSaGttxScff4ylLeheSshcvH4/Hnn8cTTv3JkxF0YQwTIhiqJgcGAAvz59GuOjo8RFJHzFBayqq8PDjz6K0rIyEq8igmUMF3FudhYXzp3D1cuXSeoDAcDyLdwHDh3CA4cOIcfjIfEqIljGQkgm0dnZiU8++gizoRAp6clSaJpGSWkpHnrkUdTU1ZKUBSJYxkXTNExPT6Pl7Fl0XLsGSRTJpGQRNrsdew4cwAOHDiE3L49YVUSwzEEykUBXZyfOnjmD0MwMsbaywKoqLS/HsYcfRnVNDSwWC5kUIljms7ZCwSAuf/YZ2ltbkYjHyaRk2gdGUXC53dh/8CD2HjgAn89HrCoiWOZGFEUMDgyg5dw5jAwOkiLqDIHnedQ1NeHQkSMIlJaC4zgyKUSwMgNd17G4uIiuzk5cPHcOs6EQSYEwKSzLoigQwOFjx1BbWwu7w0EmhQhW5rqJ4XAYba2tuN7WhoX5eRLfMgk0TSOvoAB7DxzAfbt2wUNSFYhgZQuyLCMUDKL18mV0tLeTDhBG/ogoCh6fD3v27cOuPXvgz80lCaBEsLITSZIwOTGBttZWdHd2Ir60RCwuAwmV1+/Hzt27cd+uXcgvKADLkqs+iWARIIoipqem0H71Kno6OxFdWCCTkkahyisoQGNzMxEqIliEe1lcwZkZdHZ0oOPaNUQjEdJPPkVwHIfc/Hzs3b8ftfX18Pn9RKiIYBHWgqIomJubQ293N7pv3MDUxARpY7NN2B0OlFdUoLG5GVU1NcjJyQFNkys9iWAR1o2qqojHYhgdHUVXRwcGBwYQW1wkca5NQtM0vH4/6hsb0djcjMLCQtjsdnLqRwSLsBXoug5JFDE7O4eB/j709fRgcnwckigS8VqHSDlcLpTt2IG6+nqUV1TA6/WShE8iWITtRNM0JOJxTE9Po7+3FyPDwwgHgxCSSTI5X/4AKApOtxuFxcWoqq5GVXU1cnNzYbFaiTVFBIuQDpcxtrSEYDCI0eFh9PX0YG52FqIgZGUmPUVRYBgGNocDRSUlqKuvR6C0DP5cP2w2G4lNEcEiGMVl1DQNgiBgZnoakxMTmBgfx/TkJBajUciSlLGuI03TsFit8OfmojgQQKCsDIFAAD6/HxzHEZEigkUwg9soSRJisRhmw2FMjo9jcmICwZkZxGMxU8e+VgQqx+NBSSCAktJSFBUXw+vzwW63g2VZ4u4RwSKYXcBEUUQikUBkfh6hUAihmRmEQyEsRqNYXFiAoii3RCzdy4aiqFs/VpsNOR4PPF4v8gsKkJefD39uLjxeL2w2GziOIwJFBIuQ6S6kqqqQJAnxWAyxWAyR+XnMzc0hMj+PWCyG+NISkokEJEmCLMvb4lYyDAOO58HxPCwWCxwOBxwuF1wuF3x+/7IweTxwOByw3bSeSB0fESwCAbquL8fCbgqZJEkQRRHJZBKCIEC4+b/JZBLJRAKCIEAURYiCAFmWoSrKF6w0mqbBchw4jgPH87BarbBYLLDabLDZbLDabLBarcu/tlphtdnA8zx4nr9lNa38EIhgEQjrErObv4B+m7it/Hzlz910677g4i3/4itCRASJQASLQCCYDnKeSyAQiGARCAQCESwCgUAEi0AgEIhgEQgEAhEsAoFABItAIBCIYBEIBMLm+P+gZfqrBqAXmgAAAABJRU5ErkJggg== 152 | mediatype: image/png 153 | install: 154 | spec: 155 | deployments: null 156 | strategy: "" 157 | installModes: 158 | - supported: false 159 | type: OwnNamespace 160 | - supported: false 161 | type: SingleNamespace 162 | - supported: false 163 | type: MultiNamespace 164 | - supported: true 165 | type: AllNamespaces 166 | keywords: 167 | - network 168 | - impairment 169 | - kubernetes 170 | - openshift 171 | - tc 172 | - trafficcontrol 173 | links: 174 | - name: Cluster Impairment Operator 175 | url: https://github.com/redhat-performance/cluster-impairment-operator 176 | maintainers: 177 | - email: joconnel@redhat.com 178 | name: Jared O'Connell 179 | - email: akrzos@redhat.com 180 | name: Alex Krzos 181 | maturity: beta 182 | minKubeVersion: 1.16.0 183 | provider: 184 | name: Red Hat Performance 185 | version: 1.0.0 186 | -------------------------------------------------------------------------------- /config/manifests/kustomization.yaml: -------------------------------------------------------------------------------- 1 | # These resources constitute the fully configured set of manifests 2 | # used to generate the 'manifests/' directory in a bundle. 3 | resources: 4 | - bases/cluster-impairment-operator.clusterserviceversion.yaml 5 | - ../default 6 | - ../samples 7 | - ../scorecard 8 | -------------------------------------------------------------------------------- /config/namespaced/kustomization.yaml: -------------------------------------------------------------------------------- 1 | # Adds namespace to all resources. 2 | namespace: cluster-impairment-operator-system 3 | 4 | # Value of this field is prepended to the 5 | # names of all resources, e.g. a deployment named 6 | # "wordpress" becomes "alices-wordpress". 7 | # Note that it should also match with the prefix (text before '-') of the namespace 8 | # field above. 9 | namePrefix: cluster-impairment-operator- 10 | 11 | # Labels to add to all resources and selectors. 12 | #commonLabels: 13 | # someName: someValue 14 | 15 | bases: 16 | - ../crd 17 | - ../rbac 18 | - ../manager 19 | # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. 20 | #- ../prometheus 21 | 22 | patchesStrategicMerge: 23 | # Protect the /metrics endpoint by putting it behind auth. 24 | # If you want your controller-manager to expose the /metrics 25 | # endpoint w/o any authn/z, please comment the following line. 26 | - manager_auth_proxy_patch.yaml 27 | 28 | # Mount the controller config file for loading manager configurations 29 | # through a ComponentConfig type 30 | #- manager_config_patch.yaml 31 | -------------------------------------------------------------------------------- /config/namespaced/manager_auth_proxy_patch.yaml: -------------------------------------------------------------------------------- 1 | # This patch inject a sidecar container which is a HTTP proxy for the 2 | # controller manager, it performs RBAC authorization against the Kubernetes API using SubjectAccessReviews. 3 | apiVersion: apps/v1 4 | kind: Deployment 5 | metadata: 6 | name: controller-manager 7 | namespace: system 8 | spec: 9 | template: 10 | spec: 11 | containers: 12 | - name: kube-rbac-proxy 13 | image: gcr.io/kubebuilder/kube-rbac-proxy:v0.8.0 14 | args: 15 | - "--secure-listen-address=0.0.0.0:8443" 16 | - "--upstream=http://127.0.0.1:8080/" 17 | - "--logtostderr=true" 18 | - "--v=10" 19 | ports: 20 | - containerPort: 8443 21 | name: https 22 | - name: manager 23 | args: 24 | - "--health-probe-bind-address=:6789" 25 | - "--metrics-bind-address=127.0.0.1:8080" 26 | - "--leader-elect" 27 | - "--leader-election-id=cluster-impairment-operator" 28 | - "--zap-log-level=error" 29 | -------------------------------------------------------------------------------- /config/namespaced/manager_config_patch.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: controller-manager 5 | namespace: system 6 | spec: 7 | template: 8 | spec: 9 | containers: 10 | - name: manager 11 | args: 12 | - "--config=controller_manager_config.yaml" 13 | volumeMounts: 14 | - name: manager-config 15 | mountPath: /controller_manager_config.yaml 16 | subPath: controller_manager_config.yaml 17 | volumes: 18 | - name: manager-config 19 | configMap: 20 | name: manager-config 21 | -------------------------------------------------------------------------------- /config/network-access/kustomization.yaml: -------------------------------------------------------------------------------- 1 | namespace: cluster-impairment-operator 2 | resources: 3 | - namespace.yaml 4 | - network_service_account.yaml 5 | - network_scc.yaml 6 | -------------------------------------------------------------------------------- /config/network-access/namespace.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | name: cluster-impairment-operator 5 | -------------------------------------------------------------------------------- /config/network-access/network_scc.yaml: -------------------------------------------------------------------------------- 1 | kind: SecurityContextConstraints 2 | metadata: 3 | creationTimestamp: null 4 | name: network-access-all 5 | allowHostDirVolumePlugin: true 6 | allowHostIPC: false 7 | allowHostNetwork: true 8 | allowHostPID: false 9 | allowHostPorts: false 10 | allowPrivilegedContainer: true 11 | allowedCapabilities: [] 12 | apiVersion: security.openshift.io/v1 13 | defaultAddCapabilities: [] 14 | priority: null 15 | readOnlyRootFilesystem: false 16 | requiredDropCapabilities: [] 17 | runAsUser: 18 | type: RunAsAny 19 | seLinuxContext: 20 | type: RunAsAny 21 | users: 22 | - system:serviceaccount:cluster-impairment-operator:network-access-sa 23 | -------------------------------------------------------------------------------- /config/network-access/network_service_account.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | name: network-access-sa 5 | namespace: cluster-impairment-operator 6 | -------------------------------------------------------------------------------- /config/prometheus/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - monitor.yaml 3 | -------------------------------------------------------------------------------- /config/prometheus/monitor.yaml: -------------------------------------------------------------------------------- 1 | 2 | # Prometheus Monitor Service (Metrics) 3 | apiVersion: monitoring.coreos.com/v1 4 | kind: ServiceMonitor 5 | metadata: 6 | labels: 7 | control-plane: controller-manager 8 | name: controller-manager-metrics-monitor 9 | namespace: system 10 | spec: 11 | endpoints: 12 | - path: /metrics 13 | port: https 14 | scheme: https 15 | bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token 16 | tlsConfig: 17 | insecureSkipVerify: true 18 | selector: 19 | matchLabels: 20 | control-plane: controller-manager 21 | -------------------------------------------------------------------------------- /config/rbac/auth_proxy_client_clusterrole.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRole 3 | metadata: 4 | name: metrics-reader 5 | rules: 6 | - nonResourceURLs: 7 | - "/metrics" 8 | verbs: 9 | - get 10 | -------------------------------------------------------------------------------- /config/rbac/auth_proxy_role.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRole 3 | metadata: 4 | name: proxy-role 5 | rules: 6 | - apiGroups: 7 | - authentication.k8s.io 8 | resources: 9 | - tokenreviews 10 | verbs: 11 | - create 12 | - apiGroups: 13 | - authorization.k8s.io 14 | resources: 15 | - subjectaccessreviews 16 | verbs: 17 | - create 18 | -------------------------------------------------------------------------------- /config/rbac/auth_proxy_role_binding.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRoleBinding 3 | metadata: 4 | name: proxy-rolebinding 5 | roleRef: 6 | apiGroup: rbac.authorization.k8s.io 7 | kind: ClusterRole 8 | name: proxy-role 9 | subjects: 10 | - kind: ServiceAccount 11 | name: controller-manager 12 | namespace: system 13 | -------------------------------------------------------------------------------- /config/rbac/auth_proxy_service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | labels: 5 | control-plane: controller-manager 6 | name: controller-manager-metrics-service 7 | namespace: system 8 | spec: 9 | ports: 10 | - name: https 11 | port: 8443 12 | targetPort: https 13 | selector: 14 | control-plane: controller-manager 15 | -------------------------------------------------------------------------------- /config/rbac/cluster-impairment_editor_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions for end users to edit cluster-impairments. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | name: cluster-impairment-editor-role 6 | rules: 7 | - apiGroups: 8 | - impairment.redhat.com.redhat.com 9 | resources: 10 | - cluster-impairments 11 | verbs: 12 | - create 13 | - delete 14 | - get 15 | - list 16 | - patch 17 | - update 18 | - watch 19 | - apiGroups: 20 | - impairment.redhat.com.redhat.com 21 | resources: 22 | - cluster-impairments/status 23 | verbs: 24 | - get 25 | -------------------------------------------------------------------------------- /config/rbac/cluster-impairment_viewer_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions for end users to view cluster-impairments. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | name: cluster-impairment-viewer-role 6 | rules: 7 | - apiGroups: 8 | - impairment.redhat.com.redhat.com 9 | resources: 10 | - cluster-impairments 11 | verbs: 12 | - get 13 | - list 14 | - watch 15 | - apiGroups: 16 | - impairment.redhat.com.redhat.com 17 | resources: 18 | - cluster-impairments/status 19 | verbs: 20 | - get 21 | -------------------------------------------------------------------------------- /config/rbac/clusterimpairment_editor_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions for end users to edit clusterimpairments. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | name: clusterimpairment-editor-role 6 | rules: 7 | - apiGroups: 8 | - apps.redhat.com 9 | resources: 10 | - clusterimpairments 11 | verbs: 12 | - create 13 | - delete 14 | - get 15 | - list 16 | - patch 17 | - update 18 | - watch 19 | - apiGroups: 20 | - apps.redhat.com 21 | resources: 22 | - clusterimpairments/status 23 | verbs: 24 | - get 25 | -------------------------------------------------------------------------------- /config/rbac/clusterimpairment_viewer_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions for end users to view clusterimpairments. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | name: clusterimpairment-viewer-role 6 | rules: 7 | - apiGroups: 8 | - apps.redhat.com 9 | resources: 10 | - clusterimpairments 11 | verbs: 12 | - get 13 | - list 14 | - watch 15 | - apiGroups: 16 | - apps.redhat.com 17 | resources: 18 | - clusterimpairments/status 19 | verbs: 20 | - get 21 | -------------------------------------------------------------------------------- /config/rbac/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | # All RBAC will be applied under this service account in 3 | # the deployment namespace. You may comment out this resource 4 | # if your manager will use a service account that exists at 5 | # runtime. Be sure to update RoleBinding and ClusterRoleBinding 6 | # subjects if changing service account names. 7 | - service_account.yaml 8 | - role.yaml 9 | - role_binding.yaml 10 | - leader_election_role.yaml 11 | - leader_election_role_binding.yaml 12 | # Comment the following 4 lines if you want to disable 13 | # the auth proxy (https://github.com/brancz/kube-rbac-proxy) 14 | # which protects your /metrics endpoint. 15 | - auth_proxy_service.yaml 16 | - auth_proxy_role.yaml 17 | - auth_proxy_role_binding.yaml 18 | - auth_proxy_client_clusterrole.yaml 19 | -------------------------------------------------------------------------------- /config/rbac/leader_election_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions to do leader election. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: Role 4 | metadata: 5 | name: leader-election-role 6 | rules: 7 | - apiGroups: 8 | - "" 9 | resources: 10 | - configmaps 11 | verbs: 12 | - get 13 | - list 14 | - watch 15 | - create 16 | - update 17 | - patch 18 | - delete 19 | - apiGroups: 20 | - coordination.k8s.io 21 | resources: 22 | - leases 23 | verbs: 24 | - get 25 | - list 26 | - watch 27 | - create 28 | - update 29 | - patch 30 | - delete 31 | - apiGroups: 32 | - "" 33 | resources: 34 | - events 35 | verbs: 36 | - create 37 | - patch 38 | -------------------------------------------------------------------------------- /config/rbac/leader_election_role_binding.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: RoleBinding 3 | metadata: 4 | name: leader-election-rolebinding 5 | roleRef: 6 | apiGroup: rbac.authorization.k8s.io 7 | kind: Role 8 | name: leader-election-role 9 | subjects: 10 | - kind: ServiceAccount 11 | name: controller-manager 12 | namespace: system 13 | -------------------------------------------------------------------------------- /config/rbac/role.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | name: manager-role 6 | rules: 7 | ## 8 | ## Base operator rules 9 | ## 10 | - apiGroups: 11 | - "" 12 | resources: 13 | - secrets 14 | - pods 15 | - pods/exec 16 | - pods/log 17 | verbs: 18 | - create 19 | - delete 20 | - get 21 | - list 22 | - patch 23 | - update 24 | - watch 25 | - apiGroups: 26 | - apps 27 | resources: 28 | - deployments 29 | - daemonsets 30 | - replicasets 31 | - statefulsets 32 | verbs: 33 | - create 34 | - delete 35 | - get 36 | - list 37 | - patch 38 | - update 39 | - watch 40 | ## 41 | ## Rules for impairment.redhat.com.redhat.com/v1alpha1, Kind: Cluster-impairment 42 | ## 43 | - apiGroups: 44 | - impairment.redhat.com.redhat.com 45 | resources: 46 | - cluster-impairments 47 | - cluster-impairments/status 48 | - cluster-impairments/finalizers 49 | verbs: 50 | - create 51 | - delete 52 | - get 53 | - list 54 | - patch 55 | - update 56 | - watch 57 | ## 58 | ## Rules for apps.redhat.com/v1alpha1, Kind: ClusterImpairment 59 | ## 60 | - apiGroups: 61 | - apps.redhat.com 62 | resources: 63 | - clusterimpairments 64 | - clusterimpairments/status 65 | - clusterimpairments/finalizers 66 | verbs: 67 | - create 68 | - delete 69 | - get 70 | - list 71 | - patch 72 | - update 73 | - watch 74 | - apiGroups: 75 | - "" 76 | resources: 77 | - namespaces 78 | verbs: 79 | - get 80 | - list 81 | #+kubebuilder:scaffold:rules 82 | -------------------------------------------------------------------------------- /config/rbac/role_binding.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRoleBinding 3 | metadata: 4 | name: manager-rolebinding 5 | roleRef: 6 | apiGroup: rbac.authorization.k8s.io 7 | kind: ClusterRole 8 | name: manager-role 9 | subjects: 10 | - kind: ServiceAccount 11 | name: controller-manager 12 | namespace: system 13 | -------------------------------------------------------------------------------- /config/rbac/service_account.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | name: controller-manager 5 | namespace: system 6 | -------------------------------------------------------------------------------- /config/samples/apps_v1alpha1_clusterimpairment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps.redhat.com/v1alpha1 2 | kind: ClusterImpairment 3 | metadata: 4 | name: uneven-latency 5 | spec: 6 | duration: 60 7 | startDelay: 5 8 | interfaces: 9 | - "ens2f0" 10 | ingress: 11 | latency: 10 # ms 12 | egress: 13 | latency: 100 # ms 14 | nodeSelector: 15 | key: "node-role.kubernetes.io/worker" 16 | value: "" 17 | --- 18 | apiVersion: apps.redhat.com/v1alpha1 19 | kind: ClusterImpairment 20 | metadata: 21 | name: two-min-flap 22 | spec: 23 | duration: 480 24 | startDelay: 5 25 | interfaces: 26 | - "ens2f0" 27 | linkFlapping: 28 | enable: true 29 | downTime: 120 # Seconds 30 | upTime: 120 # Seconds 31 | --- 32 | apiVersion: apps.redhat.com/v1alpha1 33 | kind: ClusterImpairment 34 | metadata: 35 | name: typical-scenario 36 | spec: 37 | duration: 30 # seconds 38 | startDelay: 5 # seconds 39 | interfaces: 40 | - "ens2f0" 41 | - "eno1" 42 | egress: 43 | latency: 50 # ms. Bidirectional, so total of 100ms 44 | ingress: 45 | latency: 50 # ms. Bidirectional, so total of 100ms 46 | loss: 0.02 # percent 47 | --- 48 | apiVersion: apps.redhat.com/v1alpha1 49 | kind: ClusterImpairment 50 | metadata: 51 | name: all-impairments 52 | spec: 53 | duration: 480 # seconds 54 | startDelay: 5 # seconds 55 | interfaces: 56 | - "ens2f0" 57 | egress: 58 | latency: 50 # ms. Bidirectional, so total of 100ms 59 | loss: 0.02 # percent 60 | bandwidth: 10000 # 1000 kbit/s, about 10 mbit/s 61 | ingress: 62 | latency: 50 # ms. Bidirectional, so total of 100ms 63 | loss: 0.02 # percent 64 | bandwidth: 10000 # 1000 kbit/s, about 10 mbit/s 65 | linkFlapping: 66 | enable: true 67 | downTime: 30 # Seconds 68 | upTime: 120 # Seconds 69 | --- 70 | apiVersion: apps.redhat.com/v1alpha1 71 | kind: ClusterImpairment 72 | metadata: 73 | name: duplication 74 | spec: 75 | duration: 60 76 | startDelay: 5 77 | interfaces: 78 | - "ens3" 79 | ingress: 80 | duplication: 25 # percent 81 | egress: 82 | duplication: 25 # percent 83 | -------------------------------------------------------------------------------- /config/samples/kustomization.yaml: -------------------------------------------------------------------------------- 1 | ## Append samples you want in your CSV to this file as resources ## 2 | resources: 3 | - apps_v1alpha1_clusterimpairment.yaml 4 | #+kubebuilder:scaffold:manifestskustomizesamples 5 | -------------------------------------------------------------------------------- /config/scorecard/bases/config.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: scorecard.operatorframework.io/v1alpha3 2 | kind: Configuration 3 | metadata: 4 | name: config 5 | stages: 6 | - parallel: true 7 | tests: [] 8 | -------------------------------------------------------------------------------- /config/scorecard/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - bases/config.yaml 3 | patchesJson6902: 4 | - path: patches/basic.config.yaml 5 | target: 6 | group: scorecard.operatorframework.io 7 | version: v1alpha3 8 | kind: Configuration 9 | name: config 10 | - path: patches/olm.config.yaml 11 | target: 12 | group: scorecard.operatorframework.io 13 | version: v1alpha3 14 | kind: Configuration 15 | name: config 16 | #+kubebuilder:scaffold:patchesJson6902 17 | -------------------------------------------------------------------------------- /config/scorecard/patches/basic.config.yaml: -------------------------------------------------------------------------------- 1 | - op: add 2 | path: /stages/0/tests/- 3 | value: 4 | entrypoint: 5 | - scorecard-test 6 | - basic-check-spec 7 | image: quay.io/operator-framework/scorecard-test:v1.8.0 8 | labels: 9 | suite: basic 10 | test: basic-check-spec-test 11 | -------------------------------------------------------------------------------- /config/scorecard/patches/olm.config.yaml: -------------------------------------------------------------------------------- 1 | - op: add 2 | path: /stages/0/tests/- 3 | value: 4 | entrypoint: 5 | - scorecard-test 6 | - olm-bundle-validation 7 | image: quay.io/operator-framework/scorecard-test:v1.8.0 8 | labels: 9 | suite: olm 10 | test: olm-bundle-validation-test 11 | - op: add 12 | path: /stages/0/tests/- 13 | value: 14 | entrypoint: 15 | - scorecard-test 16 | - olm-crds-have-validation 17 | image: quay.io/operator-framework/scorecard-test:v1.8.0 18 | labels: 19 | suite: olm 20 | test: olm-crds-have-validation-test 21 | - op: add 22 | path: /stages/0/tests/- 23 | value: 24 | entrypoint: 25 | - scorecard-test 26 | - olm-crds-have-resources 27 | image: quay.io/operator-framework/scorecard-test:v1.8.0 28 | labels: 29 | suite: olm 30 | test: olm-crds-have-resources-test 31 | - op: add 32 | path: /stages/0/tests/- 33 | value: 34 | entrypoint: 35 | - scorecard-test 36 | - olm-spec-descriptors 37 | image: quay.io/operator-framework/scorecard-test:v1.8.0 38 | labels: 39 | suite: olm 40 | test: olm-spec-descriptors-test 41 | - op: add 42 | path: /stages/0/tests/- 43 | value: 44 | entrypoint: 45 | - scorecard-test 46 | - olm-status-descriptors 47 | image: quay.io/operator-framework/scorecard-test:v1.8.0 48 | labels: 49 | suite: olm 50 | test: olm-status-descriptors-test 51 | -------------------------------------------------------------------------------- /config/testing/debug_logs_patch.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apps/v1 3 | kind: Deployment 4 | metadata: 5 | name: controller-manager 6 | namespace: system 7 | spec: 8 | template: 9 | spec: 10 | containers: 11 | - name: manager 12 | env: 13 | - name: ANSIBLE_DEBUG_LOGS 14 | value: "TRUE" 15 | -------------------------------------------------------------------------------- /config/testing/kustomization.yaml: -------------------------------------------------------------------------------- 1 | # Adds namespace to all resources. 2 | namespace: osdk-test 3 | 4 | namePrefix: osdk- 5 | 6 | # Labels to add to all resources and selectors. 7 | #commonLabels: 8 | # someName: someValue 9 | 10 | patchesStrategicMerge: 11 | - manager_image.yaml 12 | - debug_logs_patch.yaml 13 | - ../default/manager_auth_proxy_patch.yaml 14 | 15 | apiVersion: kustomize.config.k8s.io/v1beta1 16 | kind: Kustomization 17 | resources: 18 | - ../crd 19 | - ../rbac 20 | - ../manager 21 | images: 22 | - name: testing 23 | newName: testing-operator 24 | -------------------------------------------------------------------------------- /config/testing/manager_image.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apps/v1 3 | kind: Deployment 4 | metadata: 5 | name: controller-manager 6 | namespace: system 7 | spec: 8 | template: 9 | spec: 10 | containers: 11 | - name: manager 12 | image: testing 13 | -------------------------------------------------------------------------------- /config/testing/pull_policy/Always.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apps/v1 3 | kind: Deployment 4 | metadata: 5 | name: controller-manager 6 | namespace: system 7 | spec: 8 | template: 9 | spec: 10 | containers: 11 | - name: manager 12 | imagePullPolicy: Always 13 | -------------------------------------------------------------------------------- /config/testing/pull_policy/IfNotPresent.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apps/v1 3 | kind: Deployment 4 | metadata: 5 | name: controller-manager 6 | namespace: system 7 | spec: 8 | template: 9 | spec: 10 | containers: 11 | - name: manager 12 | imagePullPolicy: IfNotPresent 13 | -------------------------------------------------------------------------------- /config/testing/pull_policy/Never.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apps/v1 3 | kind: Deployment 4 | metadata: 5 | name: controller-manager 6 | namespace: system 7 | spec: 8 | template: 9 | spec: 10 | containers: 11 | - name: manager 12 | imagePullPolicy: Never 13 | -------------------------------------------------------------------------------- /container/Containerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Red Hat, Inc 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | FROM quay.io/centos/centos:8 16 | 17 | RUN yum update -y && yum install iproute-tc python3 net-tools -y 18 | 19 | COPY apply-impairments.py / 20 | 21 | ENTRYPOINT python3 apply-impairments.py 22 | -------------------------------------------------------------------------------- /container/apply-impairments.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Red Hat, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """ 16 | This python module runs impairments on the specified interface, with the 17 | specified impairments. 18 | 19 | The interface and impairments are passed in using environment variables. 20 | 21 | Variables: 22 | - DURATION: Duration in seconds as int. -1 for indefinite. Default: 60 (if no END_TIME set) 23 | - START_TIME: The timestamp (epoch) when the impairment is supposed to start. 24 | Default: Current timestamp at time script is run. 25 | - END_TIME: The timestamp (epoch) when the impairment will end. Overrides duration. 26 | - INTERFACE: Comma separated list of interfaces to apply impairments. Default: ens1f1 27 | - LINK_FLAPPING: Whether to turn on and off the interface. (True/False) Default: False 28 | - LINK_FLAP_DOWN_TIME: Time period to flap link down (Seconds) Default: 2 29 | - LINK_FLAP_UP_TIME: Time period to flap link down (Seconds). Default: 2 30 | 31 | Directional impairment variables: 32 | Uses ifb for ingress impairments (a kernel module). 33 | 0 to disable. Default 0. 34 | - EGRESS_LATENCY: Outbound latency in ms. 35 | - EGRESS_JITTER_CORRELATION: The correlation between different egress jitter values. 36 | - EGRESS_JITTER: The amount of egress jitter in milliseconds. 37 | - EGRESS_JITTER_DISTRIBUTION: The type of distribution of egress jitter. Options: normal, pareto, paretonormal 38 | - EGRESS_REORDER: The percentage of egress packets that skip the latency (causing a reorder) 39 | - EGRESS_REORDER_CORRELATION: The correlation between reorder values per egress packet. 40 | - EGRESS_DUPLICATION: The percentage of egress packets that are duplicated. 41 | - EGRESS_DUPLICATION_CORRELATION: The correlation between duplication values per egress packet. 42 | - EGRESS_CORRUPTION: The percentage of egress packets that are to be corrupted. 43 | - EGRESS_CORRUPTION_CORRELATION: The correlation between corruption values per egress packet. 44 | - EGRESS_PACKET_LOSS: Outbound packet loss. 45 | - EGRESS_LOSS_CORRELATION: The correlation between packet loss values per egress packet. 46 | - EGRESS_BANDWIDTH_LIMIT: Outbound packet loss. 47 | - INGRESS_LATENCY: Inbound latency. 48 | - INGRESS_JITTER: The amount of ingress jitter in milliseconds. 49 | - INGRESS_JITTER_CORRELATION: The correlation between different ingress jitter values. 50 | - INGRESS_JITTER_DISTRIBUTION: The type of distribution of ingress jitter. Options: normal, pareto, paretonormal 51 | - INGRESS_REORDER: The percentage of packets that skip the ingress latency (causing a reorder) 52 | - INGRESS_REORDER_CORRELATION: The correlation between reorder values per ingress packet. 53 | - INGRESS_DUPLICATION: The percentage of ingress packets that are duplicated. 54 | - INGRESS_DUPLICATION_CORRELATION: The correlation between duplication values per ingress packet. 55 | - INGRESS_CORRUPTION: The percentage of ingress packets that are to be corrupted. 56 | - INGRESS_CORRUPTION_CORRELATION: The correlation between corruption values per ingress packet. 57 | - INGRESS_PACKET_LOSS: Inbound packet loss. 58 | - INGRESS_LOSS_CORRELATION: The correlation between packet loss values per ingress packet. 59 | - INGRESS_BANDWIDTH_LIMIT: Inbound packet loss. 60 | """ 61 | 62 | 63 | import logging 64 | import os 65 | import subprocess 66 | import time 67 | import signal 68 | import sys 69 | 70 | logging.basicConfig(level=logging.INFO, format='%(asctime)s : %(levelname)s : %(message)s') 71 | logger = logging.getLogger('cluster-impairment') 72 | logging.Formatter.converter = time.gmtime 73 | 74 | running = True 75 | 76 | def on_shutdown(self, *args): 77 | global running 78 | running = False 79 | 80 | def command(cmd, dry_run, cmd_directory="", mask_output=False, mask_arg=0, no_log=False, fail_on_error=False): 81 | if cmd_directory != "": 82 | logger.debug("Command Directory: {}".format(cmd_directory)) 83 | working_directory = os.getcwd() 84 | os.chdir(cmd_directory) 85 | if dry_run: 86 | cmd.insert(0, "echo") 87 | if mask_arg == 0: 88 | logger.info("Command: {}".format(" ".join(cmd))) 89 | else: 90 | logger.info("Command: {} {} {}".format(" ".join(cmd[:mask_arg - 1]), "**(Masked)**", " ".join(cmd[mask_arg:]))) 91 | process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True) 92 | 93 | output = "" 94 | while True: 95 | output_line = process.stdout.readline() 96 | if output_line.strip() != "": 97 | if not no_log: 98 | if not mask_output: 99 | logger.info("Output : {}".format(output_line.strip())) 100 | else: 101 | logger.info("Output : **(Masked)**") 102 | if output == "": 103 | output = output_line.strip() 104 | else: 105 | output = "{}\n{}".format(output, output_line.strip()) 106 | return_code = process.poll() 107 | if return_code is not None: 108 | for output_line in process.stdout.readlines(): 109 | if output_line.strip() != "": 110 | if not no_log: 111 | if not mask_output: 112 | logger.info("Output : {}".format(output_line.strip())) 113 | else: 114 | logger.info("Output : **(Masked)**") 115 | if output == "": 116 | output = output_line 117 | else: 118 | output = "{}\n{}".format(output, output_line.strip()) 119 | logger.debug("Return Code: {}".format(return_code)) 120 | break 121 | if cmd_directory != "": 122 | os.chdir(working_directory) 123 | if fail_on_error and return_code != 0: 124 | logger.error("Error issuing command \"{}\".".format(cmd_directory)) 125 | logger.error(output) 126 | sys.exit(1) 127 | return return_code, output 128 | 129 | def parse_tc_netem_args(prefix = ""): 130 | """ 131 | Uses the environment variables to construct an array of params for netem 132 | 133 | Currently supports latency, packet loss, and bandwidth limit as documented 134 | in the module docstring. 135 | """ 136 | args = {} 137 | latency_evar = float(os.environ.get(prefix + "LATENCY", 0)) 138 | jitter_evar = int(os.environ.get(prefix + "JITTER", 0)) 139 | jitter_correlation_evar = float(os.environ.get(prefix + "JITTER_CORRELATION", 0)) 140 | jitter_distribution_evar = os.environ.get(prefix + "JITTER_DISTRIBUTION", "") 141 | packet_loss_evar = float(os.environ.get(prefix + "PACKET_LOSS", 0)) 142 | bandwidth_limit_evar = int(os.environ.get(prefix + "BANDWIDTH_LIMIT", 0)) 143 | reorder_evar = float(os.environ.get(prefix + "REORDER", 0)) 144 | reorder_correlation_evar = float(os.environ.get(prefix + "REORDER_CORRELATION", 0)) 145 | duplication_evar = float(os.environ.get(prefix + "DUPLICATION", 0)) 146 | duplication_correlation_evar = float(os.environ.get(prefix + "DUPLICATION_CORRELATION", 0)) 147 | corruption_evar = float(os.environ.get(prefix + "CORRUPTION", 0)) 148 | corruption_correlation_evar = float(os.environ.get(prefix + "CORRUPTION_CORRELATION", 0)) 149 | 150 | if latency_evar > 0: 151 | args["latency"] = ["delay", "{}ms".format(latency_evar)] 152 | if jitter_evar > 0: 153 | args["latency"].append("{}ms".format(jitter_evar)) 154 | if jitter_correlation_evar > 0: 155 | args["latency"].append("{}%".format(jitter_correlation_evar)) 156 | if jitter_distribution_evar != "": 157 | args["latency"].append("distribution") 158 | args["latency"].append(jitter_distribution_evar) 159 | if packet_loss_evar > 0: 160 | args["packet loss"] = ["loss", "{}%".format(packet_loss_evar)] 161 | if bandwidth_limit_evar > 0: 162 | args["bandwidth limit"] = ["rate", "{}kbit".format(bandwidth_limit_evar)] 163 | if corruption_evar > 0: 164 | args["corruption"] = ["corrupt", "{}%".format(corruption_evar)] 165 | if corruption_correlation_evar > 0: 166 | args["corruption"].append("{}%".format(corruption_correlation_evar)) 167 | if reorder_evar > 0: 168 | args["reorder"] = ["reorder", "{}%".format(reorder_evar)] 169 | if corruption_correlation_evar > 0: 170 | args["reorder"].append("{}%".format(reorder_correlation_evar)) 171 | if duplication_evar > 0: 172 | args["duplication"] = ["duplicate", "{}%".format(duplication_evar)] 173 | if duplication_correlation_evar > 0: 174 | args["duplication"].append("{}%".format(duplication_correlation_evar)) 175 | 176 | return args 177 | 178 | 179 | def apply_tc_netem(interfaces, impairments, dry_run=False): 180 | if len(impairments) > 1: 181 | logger.info("Applying {} impairments".format(", ".join(impairments.keys()))) 182 | elif len(impairments) == 1: 183 | logger.info("Applying only {} impairment".format(list(impairments.keys())[0])) 184 | else: 185 | logger.warn("Invalid state. Applying no impairments.") 186 | 187 | if len(interfaces) == 0: 188 | logger.warn("Invalid state. 0 interfaces given") 189 | 190 | for interface in interfaces: 191 | tc_command = ["tc", "qdisc", "add", "dev", interface, "root", "netem"] 192 | for impairment in impairments.values(): 193 | tc_command.extend(impairment) 194 | rc, _ = command(tc_command, dry_run) 195 | if rc != 0: 196 | logger.error("Applying latency and packet loss failed, tc rc: {}. Did you forget to run as privileged with host-network?".format(rc)) 197 | _, output = command(["ifconfig"], False) 198 | print(output) 199 | sys.exit(1) 200 | 201 | 202 | def remove_tc_netem(interfaces, dry_run=False, ignore_errors=False): 203 | logger.info("Removing bandwidth, latency, and packet loss impairments") 204 | for interface in interfaces: 205 | tc_command = ["tc", "qdisc", "del", "dev", interface, "root", "netem"] 206 | rc, _ = command(tc_command, dry_run) 207 | if rc != 0 and not ignore_errors: 208 | logger.error("Removing latency and packet loss failed, tc rc: {}".format(rc)) 209 | sys.exit(1) 210 | 211 | def setup_ifb(interfaces, dry_run): 212 | logger.info("Setting up ifb interface") 213 | command(["modprobe", "ifb", "numifbs=" + str(len(interfaces))], dry_run, fail_on_error=True) 214 | for i, interface in enumerate(interfaces): 215 | new_interface = "ifb" + str(i) 216 | command(["ip", "link", "set", "dev", new_interface, "up"], dry_run, fail_on_error=True) 217 | command(["tc", "qdisc", "add", "dev", interface, "ingress"], dry_run, fail_on_error=True) 218 | command(["tc", "filter", "add", "dev", interface, "parent", "ffff:", 219 | "protocol", "ip", "u32", "match", "u32", "0", "0", "flowid", 220 | "1:1", "action", "mirred", "egress", "redirect", "dev", new_interface], 221 | dry_run, fail_on_error=True) 222 | command(["tc", "filter", "add", "dev", interface, "parent", "ffff:", 223 | "protocol", "ipv6", "u32", "match", "u32", "0", "0", "flowid", 224 | "1:1", "action", "mirred", "egress", "redirect", "dev", new_interface], 225 | dry_run, fail_on_error=True) 226 | 227 | def remove_ifb(interfaces, dry_run): 228 | logger.info("Removing IFB") 229 | for i, interface in enumerate(interfaces): 230 | new_interface = "ifb" + str(i) 231 | command(["tc", "qdisc", "del", "dev", interface, "ingress"], dry_run) 232 | command(["ip", "link", "set", "dev", new_interface, "down"], dry_run) 233 | command(["modprobe", "-r", "ifb"], dry_run) 234 | 235 | def set_flap_links(interfaces, up, dry_run, ignore_errors=False): 236 | logger.info("Flapping links " + up) 237 | for interface in interfaces: 238 | ip_command = ["ip", "link", "set", interface, up] 239 | rc, _ = command(ip_command, dry_run) 240 | if rc != 0 and not ignore_errors: 241 | logger.error("RWN workload, ip link set {} {} rc: {}".format(interface, up, rc)) 242 | sys.exit(1) 243 | 244 | def main(): 245 | 246 | logger.info("Impairment script running") 247 | 248 | # It is important that this script knows when the pod is 249 | # being shut down so that the impairments can be removed. 250 | global running 251 | signal.signal(signal.SIGINT, on_shutdown) 252 | signal.signal(signal.SIGTERM, on_shutdown) 253 | 254 | # Now, the impairments 255 | 256 | ingress_netem_impairments = parse_tc_netem_args("INGRESS_") 257 | egress_netem_impairments = parse_tc_netem_args("EGRESS_") 258 | duration = int(os.environ.get("DURATION", 0)) # Seconds 259 | start_time = int(os.environ.get("START_TIME", time.time())) # Epoch 260 | end_time = int(os.environ.get("END_TIME", -1)) # Epoch 261 | egress_interfaces = os.environ.get("INTERFACE", "ens1f1").split(",") 262 | dry_run = os.environ.get("DRY_RUN", "false").lower() == "true" 263 | flap_links = os.environ.get("LINK_FLAPPING", "false").lower() == "true" 264 | link_flap_down = int(os.environ.get("LINK_FLAP_DOWN_TIME", 1)) 265 | link_flap_up = int(os.environ.get("LINK_FLAP_UP_TIME", 1)) 266 | 267 | if end_time == -1: 268 | if duration == 0: 269 | duration = 60 270 | elif duration == -1: 271 | duration = 60 * 60 * 24 * 365 # run for a year 272 | end_time = start_time + duration 273 | 274 | has_netem_impairments = (len(ingress_netem_impairments) + len(egress_netem_impairments)) != 0 275 | 276 | if has_netem_impairments or flap_links: 277 | logger.info("Running impairments") 278 | if len(ingress_netem_impairments): 279 | logger.info("Running ingress impairments") 280 | if len(egress_netem_impairments): 281 | logger.info("Running egress impairments") 282 | else: 283 | logger.warn("No impairments. Exiting") 284 | return 285 | 286 | current_time = time.time() 287 | 288 | logger.info("Set to run for {}s. End time in {}s".format(end_time - start_time, 289 | end_time - current_time)) 290 | 291 | if current_time < start_time and running: 292 | logger.info("Waiting to run impairments") 293 | while current_time < start_time and running: 294 | time.sleep(.1) 295 | current_time = time.time() 296 | 297 | if has_netem_impairments: 298 | ingress_interfaces = [] 299 | all_interfaces = [] 300 | if len(ingress_netem_impairments): 301 | for i in range(len(egress_interfaces)): 302 | ingress_interfaces.append("ifb" + str(i)) 303 | all_interfaces.extend(ingress_interfaces) 304 | if len(egress_netem_impairments): 305 | all_interfaces.extend(egress_interfaces) 306 | 307 | # Remove, just in case. 308 | remove_tc_netem( 309 | all_interfaces, 310 | dry_run, 311 | True) 312 | remove_ifb(egress_interfaces, dry_run) 313 | 314 | if len(ingress_netem_impairments): 315 | setup_ifb(egress_interfaces, dry_run) 316 | 317 | if len(ingress_netem_impairments): 318 | logger.info("Applying ingress impairments " + str(ingress_netem_impairments)) 319 | apply_tc_netem( 320 | ingress_interfaces, 321 | ingress_netem_impairments, 322 | dry_run) 323 | if len(egress_netem_impairments): 324 | logger.info("Applying egress impairments " + str(egress_netem_impairments)) 325 | apply_tc_netem( 326 | egress_interfaces, 327 | egress_netem_impairments, 328 | dry_run) 329 | else: 330 | logger.info("No netem impairments") 331 | 332 | if flap_links: 333 | link_flap_count = 1 334 | set_flap_links(egress_interfaces, "down", dry_run) 335 | next_flap_time = time.time() + link_flap_down 336 | links_down = True 337 | 338 | wait_logger = 0 339 | while current_time < end_time and running: 340 | if flap_links: 341 | if current_time >= next_flap_time: 342 | if links_down: 343 | links_down = False 344 | set_flap_links(egress_interfaces, "up", dry_run) 345 | next_flap_time = time.time() + link_flap_up 346 | else: 347 | links_down = True 348 | link_flap_count += 1 349 | set_flap_links(egress_interfaces, "down", dry_run) 350 | next_flap_time = time.time() + link_flap_down 351 | 352 | time.sleep(.1) 353 | wait_logger += 1 354 | if wait_logger >= 100 and end_time - current_time < 900: 355 | logger.info("Remaining impairment duration: {}".format(round(end_time - current_time, 1))) 356 | wait_logger = 0 357 | current_time = time.time() 358 | 359 | if not running: 360 | logger.warn("Ending early due to pod/system termination") 361 | 362 | if flap_links: 363 | set_flap_links(egress_interfaces, "up", dry_run, True) 364 | 365 | if has_netem_impairments: 366 | # Done 367 | remove_tc_netem( 368 | all_interfaces, 369 | dry_run) 370 | 371 | if len(ingress_netem_impairments): 372 | remove_ifb(egress_interfaces, dry_run) 373 | 374 | # Sleep until ended 375 | while running: 376 | time.sleep(0.1) 377 | 378 | if __name__ == '__main__': 379 | sys.exit(main()) 380 | -------------------------------------------------------------------------------- /manifests/example-1-uneven-latency-cr.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps.redhat.com/v1alpha1 2 | kind: ClusterImpairment 3 | metadata: 4 | name: uneven-latency 5 | spec: 6 | duration: 60 7 | startDelay: 5 8 | interfaces: 9 | - "ens3" 10 | ingress: 11 | latency: 10 # ms 12 | egress: 13 | latency: 100 # ms 14 | nodeSelector: 15 | key: "node-role.kubernetes.io/worker" 16 | value: "" 17 | 18 | -------------------------------------------------------------------------------- /manifests/example-2-two-min-flap-cr.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps.redhat.com/v1alpha1 2 | kind: ClusterImpairment 3 | metadata: 4 | name: two-min-flap 5 | spec: 6 | duration: 480 7 | startDelay: 5 8 | interfaces: 9 | - "ens3" 10 | linkFlapping: 11 | enable: true 12 | downTime: 12 # Seconds 13 | upTime: 12 # Seconds 14 | 15 | -------------------------------------------------------------------------------- /manifests/example-3-typical-scenario-cr.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps.redhat.com/v1alpha1 2 | kind: ClusterImpairment 3 | metadata: 4 | name: typical-scenario 5 | spec: 6 | duration: 30 # seconds 7 | startDelay: 5 # seconds 8 | interfaces: 9 | - "ens2f0" 10 | - "ens3" 11 | egress: 12 | latency: 50 # ms. Bidirectional, so total of 100ms 13 | ingress: 14 | latency: 50 # ms. Bidirectional, so total of 100ms 15 | loss: 0.02 # percent 16 | 17 | -------------------------------------------------------------------------------- /manifests/example-4-all-impairments-cr.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps.redhat.com/v1alpha1 2 | kind: ClusterImpairment 3 | metadata: 4 | name: all-impairments 5 | spec: 6 | duration: 480 # seconds 7 | startDelay: 5 # seconds 8 | interfaces: 9 | - "ens3" 10 | egress: 11 | latency: 50 # ms. Bidirectional, so total of 100ms 12 | loss: 0.02 # percent 13 | bandwidth: 10000 # 1000 kbit/s, about 10 mbit/s 14 | ingress: 15 | latency: 50 # ms. Bidirectional, so total of 100ms 16 | loss: 0.02 # percent 17 | bandwidth: 10000 # 1000 kbit/s, about 10 mbit/s 18 | linkFlapping: 19 | enable: true 20 | downTime: 30 # Seconds 21 | upTime: 120 # Seconds 22 | 23 | -------------------------------------------------------------------------------- /manifests/example-5-duplication.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps.redhat.com/v1alpha1 2 | kind: ClusterImpairment 3 | metadata: 4 | name: duplication 5 | spec: 6 | duration: 60 7 | startDelay: 5 8 | interfaces: 9 | - "ens3" 10 | ingress: 11 | duplication: 25 # percent 12 | egress: 13 | duplication: 25 # percent 14 | -------------------------------------------------------------------------------- /manifests/example-impairment-cr.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps.redhat.com/v1alpha1 2 | kind: ClusterImpairment 3 | metadata: 4 | name: test-impairment-cr 5 | spec: 6 | duration: 30 # seconds 7 | startDelay: 5 # seconds. It typically takes about 2-3 seconds for the Daemonset to run 8 | interfaces: 9 | - "ens2f0" 10 | ingress: # uses ifb 11 | bandwidth: 0 # kbit 12 | latency: 10 # ms 13 | loss: 0 # percent 14 | egress: 15 | bandwidth: 0 #kbit 16 | latency: 100 # ms 17 | loss: 0 # percent 18 | linkFlapping: 19 | enable: false 20 | downTime: 3 21 | upTime: 3 22 | nodeSelector: 23 | key: "node-role.kubernetes.io/worker" 24 | value: "" 25 | -------------------------------------------------------------------------------- /manifests/impairment-daemonset.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: DaemonSet 3 | metadata: 4 | name: cluster-impairment-initializer 5 | namespace: cluster-impairment-operator 6 | labels: 7 | k8s-app: cluster-impairment 8 | spec: 9 | selector: 10 | matchLabels: 11 | name: cluster-impairment 12 | template: 13 | metadata: 14 | labels: 15 | name: cluster-impairment 16 | spec: 17 | containers: 18 | - name: impairment-worker 19 | image: quay.io/redhat-performance/cluster-impairment-worker:latest 20 | env: 21 | - name: INTERFACE 22 | value: "ens3" 23 | - name: EGRESS_BANDWIDTH_LIMIT 24 | value: "1000" 25 | - name: DURATION 26 | value: "30" 27 | - name: EGRESS_LATENCY 28 | value: "100" 29 | # - name: EGRESS_JITTER 30 | # value: "50" 31 | # - name: "EGRESS_JITTER_DISTRIBUTION" 32 | # value: "pareto" 33 | - name: EGRESS_DUPLICATION 34 | value: "50" 35 | # - name: DRY_RUN 36 | # value: "True" 37 | securityContext: 38 | privileged: true 39 | volumeMounts: 40 | - mountPath: /lib/modules 41 | name: modprobe-modules-dir 42 | volumes: 43 | - name: modprobe-modules-dir 44 | hostPath: 45 | path: /lib/modules 46 | hostNetwork: true 47 | dnsPolicy: Default 48 | serviceAccountName: network-access 49 | -------------------------------------------------------------------------------- /manifests/impairment-pod.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | namespace: cluster-impairment-operator 5 | name: cluster-impairment-worker 6 | spec: 7 | privileged: true 8 | containers: 9 | - name: tc 10 | image: quay.io/redhat-performance/cluster-impairment-worker 11 | env: 12 | - name: INTERFACE 13 | value: "ens2f0" 14 | - name: BANDWIDTH_LIMIT 15 | value: "1000" 16 | - name: DURATION 17 | value: "60" 18 | - name: EGRESS_LATENCY 19 | value: "100" 20 | - name: DRY_RUN 21 | value: "True" 22 | securityContext: 23 | privileged: true 24 | # allowPrivilegeEscalation: true 25 | volumeMounts: 26 | - mountPath: /lib/modules 27 | name: modprobe-modules-dir 28 | restartPolicy: Never 29 | hostNetwork: true 30 | dnsPolicy: Default 31 | volumes: 32 | - name: modprobe-modules-dir 33 | hostPath: 34 | path: /lib/modules 35 | serviceAccountName: network-access 36 | -------------------------------------------------------------------------------- /manifests/serviceaccount.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | name: cluster-impairment-operator 5 | --- 6 | apiVersion: v1 7 | kind: ServiceAccount 8 | metadata: 9 | name: network-access-sa 10 | namespace: cluster-impairment-operator 11 | --- 12 | kind: SecurityContextConstraints 13 | metadata: 14 | creationTimestamp: null 15 | name: network-access-all 16 | allowHostDirVolumePlugin: true 17 | allowHostIPC: false 18 | allowHostNetwork: true 19 | allowHostPID: false 20 | allowHostPorts: false 21 | allowPrivilegedContainer: true 22 | allowedCapabilities: [] 23 | apiVersion: security.openshift.io/v1 24 | defaultAddCapabilities: [] 25 | priority: null 26 | readOnlyRootFilesystem: false 27 | requiredDropCapabilities: [] 28 | runAsUser: 29 | type: RunAsAny 30 | seLinuxContext: 31 | type: RunAsAny 32 | users: 33 | - system:serviceaccount:cluster-impairment-operator:network-access-sa 34 | -------------------------------------------------------------------------------- /media/clusterimpairment_example_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/redhat-performance/cluster-impairment-operator/726befb02f2973d730bde3738b900d591b2bcfd8/media/clusterimpairment_example_1.png -------------------------------------------------------------------------------- /media/icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/redhat-performance/cluster-impairment-operator/726befb02f2973d730bde3738b900d591b2bcfd8/media/icon.png -------------------------------------------------------------------------------- /media/simple-example-1.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/redhat-performance/cluster-impairment-operator/726befb02f2973d730bde3738b900d591b2bcfd8/media/simple-example-1.gif -------------------------------------------------------------------------------- /molecule/default/converge.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Converge 3 | hosts: localhost 4 | connection: local 5 | gather_facts: no 6 | collections: 7 | - community.kubernetes 8 | 9 | tasks: 10 | - name: Create Namespace 11 | k8s: 12 | api_version: v1 13 | kind: Namespace 14 | name: '{{ namespace }}' 15 | 16 | - import_tasks: kustomize.yml 17 | vars: 18 | state: present 19 | -------------------------------------------------------------------------------- /molecule/default/create.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Create 3 | hosts: localhost 4 | connection: local 5 | gather_facts: false 6 | tasks: [] 7 | -------------------------------------------------------------------------------- /molecule/default/destroy.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Destroy 3 | hosts: localhost 4 | connection: local 5 | gather_facts: false 6 | collections: 7 | - community.kubernetes 8 | 9 | tasks: 10 | - import_tasks: kustomize.yml 11 | vars: 12 | state: absent 13 | 14 | - name: Destroy Namespace 15 | k8s: 16 | api_version: v1 17 | kind: Namespace 18 | name: '{{ namespace }}' 19 | state: absent 20 | 21 | - name: Unset pull policy 22 | command: '{{ kustomize }} edit remove patch pull_policy/{{ operator_pull_policy }}.yaml' 23 | args: 24 | chdir: '{{ config_dir }}/testing' 25 | -------------------------------------------------------------------------------- /molecule/default/kustomize.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Build kustomize testing overlay 3 | # load_restrictor must be set to none so we can load patch files from the default overlay 4 | command: '{{ kustomize }} build --load_restrictor none .' 5 | args: 6 | chdir: '{{ config_dir }}/testing' 7 | register: resources 8 | changed_when: false 9 | 10 | - name: Set resources to {{ state }} 11 | k8s: 12 | definition: '{{ item }}' 13 | state: '{{ state }}' 14 | wait: yes 15 | loop: '{{ resources.stdout | from_yaml_all | list }}' 16 | -------------------------------------------------------------------------------- /molecule/default/molecule.yml: -------------------------------------------------------------------------------- 1 | --- 2 | dependency: 3 | name: galaxy 4 | driver: 5 | name: delegated 6 | lint: | 7 | set -e 8 | yamllint -d "{extends: relaxed, rules: {line-length: {max: 120}}}" . 9 | platforms: 10 | - name: cluster 11 | groups: 12 | - k8s 13 | provisioner: 14 | name: ansible 15 | lint: | 16 | set -e 17 | ansible-lint 18 | inventory: 19 | group_vars: 20 | all: 21 | namespace: ${TEST_OPERATOR_NAMESPACE:-osdk-test} 22 | host_vars: 23 | localhost: 24 | ansible_python_interpreter: '{{ ansible_playbook_python }}' 25 | config_dir: ${MOLECULE_PROJECT_DIRECTORY}/config 26 | samples_dir: ${MOLECULE_PROJECT_DIRECTORY}/config/samples 27 | operator_image: ${OPERATOR_IMAGE:-""} 28 | operator_pull_policy: ${OPERATOR_PULL_POLICY:-"Always"} 29 | kustomize: ${KUSTOMIZE_PATH:-kustomize} 30 | env: 31 | K8S_AUTH_KUBECONFIG: ${KUBECONFIG:-"~/.kube/config"} 32 | verifier: 33 | name: ansible 34 | lint: | 35 | set -e 36 | ansible-lint 37 | -------------------------------------------------------------------------------- /molecule/default/prepare.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Prepare 3 | hosts: localhost 4 | connection: local 5 | gather_facts: false 6 | 7 | tasks: 8 | - name: Ensure operator image is set 9 | fail: 10 | msg: | 11 | You must specify the OPERATOR_IMAGE environment variable in order to run the 12 | 'default' scenario 13 | when: not operator_image 14 | 15 | - name: Set testing image 16 | command: '{{ kustomize }} edit set image testing={{ operator_image }}' 17 | args: 18 | chdir: '{{ config_dir }}/testing' 19 | 20 | - name: Set pull policy 21 | command: '{{ kustomize }} edit add patch pull_policy/{{ operator_pull_policy }}.yaml' 22 | args: 23 | chdir: '{{ config_dir }}/testing' 24 | 25 | - name: Set testing namespace 26 | command: '{{ kustomize }} edit set namespace {{ namespace }}' 27 | args: 28 | chdir: '{{ config_dir }}/testing' 29 | -------------------------------------------------------------------------------- /molecule/default/tasks/cluster-impairment_test.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Create the impairment.redhat.com.redhat.com/v1alpha1.Cluster-impairment 3 | k8s: 4 | state: present 5 | namespace: '{{ namespace }}' 6 | definition: "{{ lookup('template', '/'.join([samples_dir, cr_file])) | from_yaml }}" 7 | wait: yes 8 | wait_timeout: 300 9 | wait_condition: 10 | type: Running 11 | reason: Successful 12 | status: "True" 13 | vars: 14 | cr_file: 'impairment.redhat.com_v1alpha1_cluster-impairment.yaml' 15 | 16 | - name: Add assertions here 17 | assert: 18 | that: false 19 | fail_msg: FIXME Add real assertions for your operator 20 | -------------------------------------------------------------------------------- /molecule/default/tasks/clusterimpairment_test.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Create the apps.redhat.com/v1alpha1.ClusterImpairment 3 | k8s: 4 | state: present 5 | namespace: '{{ namespace }}' 6 | definition: "{{ lookup('template', '/'.join([samples_dir, cr_file])) | from_yaml }}" 7 | wait: yes 8 | wait_timeout: 300 9 | wait_condition: 10 | type: Running 11 | reason: Successful 12 | status: "True" 13 | vars: 14 | cr_file: 'apps_v1alpha1_clusterimpairment.yaml' 15 | 16 | - name: Add assertions here 17 | assert: 18 | that: false 19 | fail_msg: FIXME Add real assertions for your operator 20 | -------------------------------------------------------------------------------- /molecule/default/verify.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Verify 3 | hosts: localhost 4 | connection: local 5 | gather_facts: no 6 | collections: 7 | - community.kubernetes 8 | 9 | vars: 10 | ctrl_label: control-plane=controller-manager 11 | 12 | tasks: 13 | - block: 14 | - name: Import all test files from tasks/ 15 | include_tasks: '{{ item }}' 16 | with_fileglob: 17 | - tasks/*_test.yml 18 | rescue: 19 | - name: Retrieve relevant resources 20 | k8s_info: 21 | api_version: '{{ item.api_version }}' 22 | kind: '{{ item.kind }}' 23 | namespace: '{{ namespace }}' 24 | loop: 25 | - api_version: v1 26 | kind: Pod 27 | - api_version: apps/v1 28 | kind: Deployment 29 | - api_version: v1 30 | kind: Secret 31 | - api_version: v1 32 | kind: ConfigMap 33 | register: debug_resources 34 | 35 | - name: Retrieve Pod logs 36 | k8s_log: 37 | name: '{{ item.metadata.name }}' 38 | namespace: '{{ namespace }}' 39 | container: manager 40 | loop: "{{ q('k8s', api_version='v1', kind='Pod', namespace=namespace, label_selector=ctrl_label) }}" 41 | register: debug_logs 42 | 43 | - name: Output gathered resources 44 | debug: 45 | var: debug_resources 46 | 47 | - name: Output gathered logs 48 | debug: 49 | var: item.log_lines 50 | loop: '{{ debug_logs.results }}' 51 | 52 | - name: Re-emit failure 53 | vars: 54 | failed_task: 55 | result: '{{ ansible_failed_result }}' 56 | fail: 57 | msg: '{{ failed_task }}' 58 | -------------------------------------------------------------------------------- /molecule/kind/converge.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Converge 3 | hosts: localhost 4 | connection: local 5 | gather_facts: no 6 | 7 | tasks: 8 | - name: Build operator image 9 | docker_image: 10 | build: 11 | path: '{{ project_dir }}' 12 | pull: no 13 | name: '{{ operator_image }}' 14 | tag: latest 15 | push: no 16 | source: build 17 | force_source: yes 18 | 19 | - name: Load image into kind cluster 20 | command: kind load docker-image --name osdk-test '{{ operator_image }}' 21 | register: result 22 | changed_when: '"not yet present" in result.stdout' 23 | 24 | - import_playbook: ../default/converge.yml 25 | -------------------------------------------------------------------------------- /molecule/kind/create.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Create 3 | hosts: localhost 4 | connection: local 5 | gather_facts: false 6 | tasks: 7 | - name: Create test kind cluster 8 | command: kind create cluster --name osdk-test --kubeconfig {{ kubeconfig }} 9 | -------------------------------------------------------------------------------- /molecule/kind/destroy.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Destroy 3 | hosts: localhost 4 | connection: local 5 | gather_facts: false 6 | collections: 7 | - community.kubernetes 8 | 9 | tasks: 10 | - name: Destroy test kind cluster 11 | command: kind delete cluster --name osdk-test --kubeconfig {{ kubeconfig }} 12 | 13 | - name: Unset pull policy 14 | command: '{{ kustomize }} edit remove patch pull_policy/{{ operator_pull_policy }}.yaml' 15 | args: 16 | chdir: '{{ config_dir }}/testing' 17 | -------------------------------------------------------------------------------- /molecule/kind/molecule.yml: -------------------------------------------------------------------------------- 1 | --- 2 | dependency: 3 | name: galaxy 4 | driver: 5 | name: delegated 6 | lint: | 7 | set -e 8 | yamllint -d "{extends: relaxed, rules: {line-length: {max: 120}}}" . 9 | platforms: 10 | - name: cluster 11 | groups: 12 | - k8s 13 | provisioner: 14 | name: ansible 15 | playbooks: 16 | prepare: ../default/prepare.yml 17 | verify: ../default/verify.yml 18 | lint: | 19 | set -e 20 | ansible-lint 21 | inventory: 22 | group_vars: 23 | all: 24 | namespace: ${TEST_OPERATOR_NAMESPACE:-osdk-test} 25 | host_vars: 26 | localhost: 27 | ansible_python_interpreter: '{{ ansible_playbook_python }}' 28 | config_dir: ${MOLECULE_PROJECT_DIRECTORY}/config 29 | samples_dir: ${MOLECULE_PROJECT_DIRECTORY}/config/samples 30 | project_dir: ${MOLECULE_PROJECT_DIRECTORY} 31 | operator_image: testing-operator 32 | operator_pull_policy: "Never" 33 | kubeconfig: "{{ lookup('env', 'KUBECONFIG') }}" 34 | kustomize: ${KUSTOMIZE_PATH:-kustomize} 35 | env: 36 | K8S_AUTH_KUBECONFIG: ${MOLECULE_EPHEMERAL_DIRECTORY}/kubeconfig 37 | KUBECONFIG: ${MOLECULE_EPHEMERAL_DIRECTORY}/kubeconfig 38 | verifier: 39 | name: ansible 40 | lint: | 41 | set -e 42 | ansible-lint 43 | -------------------------------------------------------------------------------- /operatorhub_description.txt: -------------------------------------------------------------------------------- 1 | | 2 | cluster-impairment-operator is an operator designed to automate the application of impairments to the nodes in a cluster. 3 | 4 | ## Features 5 | 6 | Traffic Direction: 7 | 8 | * Egress impairments affect the traffic going out. 9 | * Ingress impairments affect the traffic coming in. (Ingress impairments require the kernel module IFB.) 10 | 11 | ### Impairments 12 | 13 | | Impairment | Description | Unit | Uses Netem | Correlation Supported | 14 | |---------------|--------------------------------------------|---------|------------|-----------------------| 15 | | Bandwidth | The bandwidth limit | kbit/s | Yes | No | 16 | | Latency | The delay of the packets | ms | Yes | Yes | 17 | | Packet Loss | The percent of packets that are dropped | percent | Yes | Yes | 18 | | Corruption | The percent of packets that are corrupted | percent | Yes | Yes | 19 | | Duplication | The percent of packets that are duplicated | percent | Yes | Yes | 20 | | Link Flapping | Turns the interface on and off | bool | No | No | 21 | 22 | 23 | On the tested environment (RHEL CoreOS 48.84), the impairments can be used alongside link flapping. 24 | 25 | 26 | #### Latency Options 27 | 28 | In addition to simply delaying the packets, there are advanced latency options. 29 | 30 | ##### Jitter 31 | 32 | Semi-randomly adds or subtracts from the latency according to the distribution up to the amount specified. 33 | If latency is 100ms, and jitter is 10ms, the actual latency will vary 100±10ms. 34 | 35 | 36 | ##### Distribution 37 | 38 | The distribution of the jitter. The options are: 39 | * Normal 40 | * Uniform 41 | * Pareto 42 | * Paretonormal 43 | 44 | ##### Jitter correlation 45 | 46 | The percent chance that the next latency's jitter value will correlate with the preceeding latency. 47 | 48 | ##### Reorder 49 | 50 | The percentage of packets that are not delayed, causing a reorder due to them being sent before the delayed ones. 51 | Applying jitter itself has the potential to also cause reordering. 52 | 53 | ##### Reorder correlation 54 | 55 | The percet chance that the value for the next reorder will correlate with the preceeding value. 56 | 57 | #### Loss options 58 | 59 | ##### Correlation 60 | 61 | The percent chance that the previous loss value for a packet correlates with the loss value for the next packet. 62 | 63 | #### Corruption options 64 | 65 | ###### Corruption 66 | 67 | The percent chance that each packet will be corrupted. 68 | 69 | ##### Corruption correlation 70 | 71 | The percet chance that the value for the next corruption will correlate with the preceeding value. 72 | 73 | #### Duplication options 74 | 75 | ##### Duplication 76 | 77 | The percent chance that each packet will be duplicated. 78 | 79 | ##### Duplication correlation 80 | 81 | The percet chance that the value for the next duplication will correlate with the preceeding value. 82 | 83 | 84 | 85 | ## Configuration 86 | 87 | Here is an example of the ClusterImpairment custom resource. 88 | 89 | 90 | ``` 91 | 92 | apiVersion: apps.redhat.com/v1alpha1 93 | kind: ClusterImpairment 94 | metadata: 95 | name: test-impairment-cr 96 | spec: 97 | duration: 30 # seconds 98 | startDelay: 5 # seconds. It typically takes about 2-3 seconds for the Daemonset to run 99 | interfaces: 100 | - "ens2f0" 101 | ingress: # uses ifb 102 | bandwidth: 0 # kbit 103 | latency: 10 # ms 104 | latencyOptions: 105 | jitter: 5 # ms 106 | jitterCorrelation: 25 # percent 107 | distribution: normal 108 | reorder: 25 # percent of packets that will skip the delay 109 | reorderCorrelation: 25 # percent 110 | loss: 0 # percent 111 | lossOptions: 112 | correlation: 25 # percent 113 | corruption: 0.1 # percent 114 | corruptionOptions: 115 | correlation: 25 # percent 116 | duplication: 1 # percent 117 | duplicationOptions: 118 | correlation: 25 # percent 119 | egress: 120 | bandwidth: 0 # kbit 121 | latency: 100 # ms 122 | loss: 0 # percent 123 | linkFlapping: 124 | enable: false 125 | downTime: 3 # Seconds 126 | upTime: 3 # Seconds 127 | nodeSelector: 128 | key: "node-role.kubernetes.io/worker" 129 | value: "" 130 | 131 | ``` 132 | 133 | #### Interfaces 134 | 135 | The interfaces configuration option is a list of all interfaces that the impairments should be applied to. The current implementation will error out once it hits an invalid interface. 136 | 137 | If an invalid interface is found, it will print out the list of interfaces. 138 | 139 | #### Node Selector 140 | 141 | The default node selector is all worker nodes, but you can change it to whatever node selector you want by setting the key and value. 142 | There is a toleration to allow it to run on master nodes if the node selector includes master nodes. 143 | 144 | #### Duration 145 | 146 | The number of seconds the impairments should run. It will try to sync the start and end time between all pods. 147 | If set to -1, it will run indefinitely (a year), until you remove the ClusterImpairment custom resource. This is good for testing that requires steady impairments. 148 | 149 | If the script is link flapping, set this value to be short enough so that if the link flap interrupts communication between the nodes, the daemonset will remove itself. 150 | 151 | #### Start Delay 152 | 153 | The delay before starting the script. If you want the pods to be in sync, a minimum of a few seconds should be used because the pods take time to start up. 154 | 155 | You can also utilize this feature to run an separate impairment after another. Just apply two resources at the same time, but add the duration and start delay of the first to the start delay of the second. 156 | 157 | #### Ingress and Egress 158 | 159 | The configuration sections "ingress" and "egress" apply to each direction. 160 | 161 | ##### Examples: 162 | 163 | ##### Example 1 164 | 165 | 166 | In this example, egress latency is set to 100ms, but the ingress latency is set to 10ms. So the latency to the interface will end up being 10ms, but 100ms going out. When pinging, this will result in 110ms of latency. 167 | 168 | ``` 169 | 170 | apiVersion: apps.redhat.com/v1alpha1 171 | kind: ClusterImpairment 172 | metadata: 173 | name: uneven-latency 174 | spec: 175 | duration: 60 176 | startDelay: 5 177 | interfaces: 178 | - "ens2f0" 179 | ingress: 180 | latency: 10 # ms 181 | egress: 182 | latency: 100 # ms 183 | 184 | ``` 185 | 186 | ##### Example 2 187 | 188 | 189 | In this example, link flapping will be enabled, and it will turn the interface on and off every 2 minutes. That is long enough for kubernetes to determine that a node is unavailable. 190 | 191 | ``` 192 | 193 | apiVersion: apps.redhat.com/v1alpha1 194 | kind: ClusterImpairment 195 | metadata: 196 | name: two-min-flap 197 | spec: 198 | duration: 480 199 | startDelay: 5 200 | interfaces: 201 | - "ens2f0" 202 | linkFlapping: 203 | enable: true 204 | downTime: 120 # Seconds 205 | upTime: 120 # Seconds 206 | 207 | ``` 208 | 209 | ##### Example 3 210 | 211 | 212 | In this example, a realistic set of impairments is applied to `ens2f0` and `eno1` for 30 seconds: 213 | 214 | ``` 215 | 216 | apiVersion: apps.redhat.com/v1alpha1 217 | kind: ClusterImpairment 218 | metadata: 219 | name: typical-scenario 220 | spec: 221 | duration: 30 # seconds 222 | startDelay: 5 # seconds 223 | interfaces: 224 | - "ens2f0" 225 | - "eno1" 226 | egress: 227 | latency: 50 # ms. Bidirectional, so total of 100ms 228 | ingress: 229 | latency: 50 # ms. Bidirectional, so total of 100ms 230 | loss: 0.02 # percent 231 | 232 | ``` 233 | 234 | ##### Example 4 235 | 236 | 237 | In this example, a combination of latency, packet loss, bandwidth, and link flapping is applied. 238 | 239 | ``` 240 | 241 | apiVersion: apps.redhat.com/v1alpha1 242 | kind: ClusterImpairment 243 | metadata: 244 | name: all-impairments 245 | spec: 246 | duration: 480 # seconds 247 | startDelay: 5 # seconds 248 | interfaces: 249 | - "ens2f0" 250 | egress: 251 | latency: 50 # ms. Bidirectional, so total of 100ms 252 | loss: 0.02 # percent 253 | bandwidth: 1000 # 1000 kbit/s, about 1 mbit/s 254 | ingress: 255 | latency: 50 # ms. Bidirectional, so total of 100ms 256 | loss: 0.02 # percent 257 | bandwidth: 1000 # 1000 kbit/s, about 1 mbit/s 258 | linkFlapping: 259 | enable: true 260 | downTime: 30 # Seconds 261 | upTime: 120 # Seconds 262 | 263 | ``` 264 | 265 | ##### Example 5 266 | 267 | 268 | In this example, packet duplicaiton is applied. 269 | 270 | 271 | ``` 272 | 273 | apiVersion: apps.redhat.com/v1alpha1 274 | kind: ClusterImpairment 275 | metadata: 276 | name: duplication 277 | spec: 278 | duration: 60 279 | startDelay: 5 280 | interfaces: 281 | - "ens3" 282 | ingress: 283 | duplication: 25 # percent 284 | egress: 285 | duplication: 25 # percent 286 | 287 | ``` 288 | 289 | ## Setup 290 | 291 | ### Requirements 292 | 293 | 1. You need `make` installed. 294 | 2. You need access to the kubernetes cluster with a kubeconfig. 295 | 296 | ### Installation 297 | 298 | To run using the current latest image: 299 | 1. Clone the repository. 300 | 2. Run `make deploy` with KUBECONFIG environment variable set to the location of the kubeconfig. 301 | 302 | To run with your own code, there are more steps. 303 | 1. Fork the repository. 304 | 2. Clone to a machine that has access to the Kubernetes cluster and the kubeconfig. 305 | 3. Modify the makefile to change the `IMG` variable to your image repository. If you do not have podman installed, also change podman to docker. 306 | 4. Run `make docker-build` then `make docker-push`. 307 | 5. Then run `make deploy` 308 | 309 | ### Deploying from operator-hub 310 | 311 | *Important* You must manually apply the serviceaccount! 312 | 313 | First, Follow the instructions on this site to deploy from here. 314 | 315 | Next, apply the manifest "serviceaccount.yaml" from the manifests folder. 316 | 317 | Then you can apply your ClusterImpairment resources. 318 | 319 | ## Running impairments 320 | 321 | First, configure a ClusterImpairment type's spec with the information for the impairment you want to run. Most importantly, set the interface(s). You can list the interfaces with `ifconfig`. If an invalid interface is set, the pod's logs will show `ifconfig` output. 322 | 323 | 324 | Once the clusterimpairment type is set, apply it and it will work. 325 | 326 | 327 | To re-run an impairment, delete the impairment with `oc delete -f path/to/manifest.yaml`, then apply it again. 328 | 329 | 330 | ## Limitations 331 | 332 | 333 | ### Multiple Impairments 334 | 335 | 336 | You should avoid any configurations that apply impairments to the same interface on the same node. There are potential conflicts. The worker pod will attempt to remove all impairments on that interface before applying new ones. 337 | 338 | 339 | Instead, take advantage of the full control of both ingress and egress impairments from within the same ClusterImpairment resource. 340 | 341 | 342 | There is currently a configured limit of 20 ClusterImpairment resources working at the same time. If that is a limitation for your use case, you can change it in `config/manager/manager.yaml` or you can try to find a way to make each ClusterImpairment do more. 343 | 344 | ### Traffic Control (TC) 345 | 346 | Traffic control is how cluster-impairment-operator applies the latency, bandwidth, and packet loss impairments. The limitation is due to Linux not being realtime operating system, so the impairment will not be perfectly consistent. 347 | 348 | ### Link Flapping 349 | 350 | When link flapping, if you flap the link that Kubernetes uses to communicate with the pods, you may be unable to remove the pod until the link is on long enough for Kubernetes to communicate with the impaired node. 351 | 352 | 353 | In this case, it is helpful to set the duration properly instead of running for an indefinite or large amount of time, because the node will properly go back to the unimpaired state at that time. 354 | 355 | 356 | If the cluster becomes offline due to the link flapping when you do not want it to be offline, soft restarting the nodes after removing the custom resource should remove all impairments. 357 | -------------------------------------------------------------------------------- /playbooks/.placeholder: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/redhat-performance/cluster-impairment-operator/726befb02f2973d730bde3738b900d591b2bcfd8/playbooks/.placeholder -------------------------------------------------------------------------------- /playbooks/clusterimpairment.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - hosts: localhost 3 | gather_facts: yes 4 | collections: 5 | - community.kubernetes 6 | - operator_sdk.util 7 | tasks: [] 8 | roles: 9 | - deploy-impairment-workers 10 | 11 | -------------------------------------------------------------------------------- /playbooks/roles/.placeholder: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/redhat-performance/cluster-impairment-operator/726befb02f2973d730bde3738b900d591b2bcfd8/playbooks/roles/.placeholder -------------------------------------------------------------------------------- /playbooks/roles/deploy-impairment-workers/.travis.yml: -------------------------------------------------------------------------------- 1 | --- 2 | language: python 3 | python: "2.7" 4 | 5 | # Use the new container infrastructure 6 | sudo: false 7 | 8 | # Install ansible 9 | addons: 10 | apt: 11 | packages: 12 | - python-pip 13 | 14 | install: 15 | # Install ansible 16 | - pip install ansible 17 | 18 | # Check ansible version 19 | - ansible --version 20 | 21 | # Create ansible.cfg with correct roles_path 22 | - printf '[defaults]\nroles_path=../' >ansible.cfg 23 | 24 | script: 25 | # Basic role syntax check 26 | - ansible-playbook tests/test.yml -i tests/inventory --syntax-check 27 | 28 | notifications: 29 | webhooks: https://galaxy.ansible.com/api/v1/notifications/ -------------------------------------------------------------------------------- /playbooks/roles/deploy-impairment-workers/README.md: -------------------------------------------------------------------------------- 1 | Role Name 2 | ========= 3 | 4 | A brief description of the role goes here. 5 | 6 | Requirements 7 | ------------ 8 | 9 | Any pre-requisites that may not be covered by Ansible itself or the role should be mentioned here. For instance, if the role uses the EC2 module, it may be a good idea to mention in this section that the boto package is required. 10 | 11 | Role Variables 12 | -------------- 13 | 14 | A description of the settable variables for this role should go here, including any variables that are in defaults/main.yml, vars/main.yml, and any variables that can/should be set via parameters to the role. Any variables that are read from other roles and/or the global scope (ie. hostvars, group vars, etc.) should be mentioned here as well. 15 | 16 | Dependencies 17 | ------------ 18 | 19 | A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles. 20 | 21 | Example Playbook 22 | ---------------- 23 | 24 | Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too: 25 | 26 | - hosts: servers 27 | roles: 28 | - { role: username.rolename, x: 42 } 29 | 30 | License 31 | ------- 32 | 33 | BSD 34 | 35 | Author Information 36 | ------------------ 37 | 38 | An optional section for the role authors to include contact information, or a website (HTML is not allowed). 39 | -------------------------------------------------------------------------------- /playbooks/roles/deploy-impairment-workers/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # defaults file for deploy-impairment-workers 3 | ansible_python_interpreter: /usr/bin/python3.8 4 | 5 | duration: 30 # seconds 6 | startDelay: 5 # seconds. It typically takes about 2-3 seconds for the Daemonset to run 7 | interfaces: 8 | - "ens2f0" 9 | 10 | # Both ingress and egress 11 | bandwidth: 0 # kbit 12 | latency: 0 # latency 13 | loss: 0 # percent 14 | 15 | # ingress and egress overwrite the bidirectional 16 | ingress: # uses ifb 17 | bandwidth: 0 # kbit 18 | latency: 0 # ms 19 | loss: 0 # percent 20 | egress: 21 | bandwidth: 0 #kbit 22 | latency: 0 # ms 23 | loss: 0 # percent 24 | 25 | linkFlapping: 26 | enable: false 27 | downTime: 3 28 | upTime: 3 29 | 30 | -------------------------------------------------------------------------------- /playbooks/roles/deploy-impairment-workers/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # handlers file for deploy-impairment-workers -------------------------------------------------------------------------------- /playbooks/roles/deploy-impairment-workers/meta/main.yml: -------------------------------------------------------------------------------- 1 | galaxy_info: 2 | author: your name 3 | description: your role description 4 | company: your company (optional) 5 | 6 | # If the issue tracker for your role is not on github, uncomment the 7 | # next line and provide a value 8 | # issue_tracker_url: http://example.com/issue/tracker 9 | 10 | # Choose a valid license ID from https://spdx.org - some suggested licenses: 11 | # - BSD-3-Clause (default) 12 | # - MIT 13 | # - GPL-2.0-or-later 14 | # - GPL-3.0-only 15 | # - Apache-2.0 16 | # - CC-BY-4.0 17 | license: license (GPL-2.0-or-later, MIT, etc) 18 | 19 | min_ansible_version: 2.9 20 | 21 | # If this a Container Enabled role, provide the minimum Ansible Container version. 22 | # min_ansible_container_version: 23 | 24 | # 25 | # Provide a list of supported platforms, and for each platform a list of versions. 26 | # If you don't wish to enumerate all versions for a particular platform, use 'all'. 27 | # To view available platforms and versions (or releases), visit: 28 | # https://galaxy.ansible.com/api/v1/platforms/ 29 | # 30 | # platforms: 31 | # - name: Fedora 32 | # versions: 33 | # - all 34 | # - 25 35 | # - name: SomePlatform 36 | # versions: 37 | # - all 38 | # - 1.0 39 | # - 7 40 | # - 99.99 41 | 42 | galaxy_tags: [] 43 | # List tags for your role here, one per line. A tag is a keyword that describes 44 | # and categorizes the role. Users find roles by searching for tags. Be sure to 45 | # remove the '[]' above, if you add tags to this list. 46 | # 47 | # NOTE: A tag is limited to a single word comprised of alphanumeric characters. 48 | # Maximum 20 tags per role. 49 | 50 | dependencies: [] 51 | # List your role dependencies here, one per line. Be sure to remove the '[]' above, 52 | # if you add dependencies to this list. 53 | -------------------------------------------------------------------------------- /playbooks/roles/deploy-impairment-workers/tasks/main.yml: -------------------------------------------------------------------------------- 1 | #--- 2 | # tasks file for deploy-impairment-workers 3 | # Copyright 2021 Red Hat, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | - debug: 18 | msg: "Running main task for ClusterImpairment '{{ ansible_operator_meta.name }}' in namespace {{ ansible_operator_meta.namespace }}" 19 | 20 | - name: Get ClusterImpairment status 21 | k8s_info: 22 | api_version: apps.redhat.com/v1alpha1 23 | kind: ClusterImpairment 24 | name: "{{ ansible_operator_meta.name }}" 25 | namespace: "{{ ansible_operator_meta.namespace }}" 26 | register: impairment_facts 27 | 28 | - name: Set name of daemonset 29 | set_fact: 30 | daemonset_name: "cluster-impairment-daemonset-{{ ansible_operator_meta.name }}" 31 | 32 | - name: Get namespace status 33 | k8s_info: 34 | api_version: v1 35 | kind: Namespace 36 | name: "cluster-impairment-operator" 37 | namespace: "cluster-impairment-operator" 38 | register: namespace_facts 39 | 40 | - name: Run impairments 41 | when: "(namespace_facts.resources|length) > 0 and impairment_facts.resources is defined and (impairment_facts.resources|length) > 0 and (('status' not in impairment_facts.resources[0]) or 'run_status' not in impairment_facts.resources[0]['status'] or impairment_facts.resources[0]['status']['run_status'] == 'not_run')" 42 | block: 43 | - name: Mark as starting 44 | operator_sdk.util.k8s_status: 45 | api_version: apps.redhat.com/v1alpha1 46 | kind: ClusterImpairment 47 | name: "{{ ansible_operator_meta.name }}" 48 | namespace: "{{ ansible_operator_meta.namespace }}" 49 | status: 50 | run_status: "starting" 51 | 52 | - name: Install dependencies 53 | pip: 54 | name: 55 | - openshift 56 | - kubernetes 57 | 58 | - name: Calculate start time epoch 59 | set_fact: 60 | start_time_epoch: "{{ (ansible_date_time.epoch | int) + (start_delay | int) }}" 61 | 62 | - name: Calculate end time epoch 63 | set_fact: 64 | end_time_epoch: "{{ (start_time_epoch | int) + (duration | int) }}" 65 | when: duration != -1 66 | 67 | - name: Create the list of interfaces 68 | set_fact: 69 | impairment_interface_list: "{{ interfaces | join(',') }}" 70 | 71 | - name: Deploy Daemonset 72 | k8s: 73 | definition: "{{ lookup('template', 'impairment-daemonset.yaml.j2') | from_yaml }}" 74 | wait: yes 75 | wait_timeout: 12 76 | 77 | - name: Mark as running 78 | operator_sdk.util.k8s_status: 79 | api_version: apps.redhat.com/v1alpha1 80 | kind: ClusterImpairment 81 | name: "{{ ansible_operator_meta.name }}" 82 | namespace: "{{ ansible_operator_meta.namespace }}" 83 | status: 84 | run_status: "running" 85 | 86 | - name: Calculate total sleep time 87 | set_fact: 88 | sleep_time: "{{ duration + start_delay }}" 89 | when: "{{ duration|int >= 0 }}" 90 | 91 | - name: Calculate total sleep time with indefinite time 92 | set_fact: 93 | sleep_time: 31536000 # a year 94 | when: "{{ duration|int == -1 }}" 95 | 96 | - name: Wait for pods to finish 97 | k8s_info: 98 | api_version: apps.redhat.com/v1alpha1 99 | kind: ClusterImpairment 100 | name: "{{ ansible_operator_meta.name }}" 101 | namespace: "{{ ansible_operator_meta.namespace }}" 102 | register: impairment_facts 103 | # Go until either the resource is deleted 104 | until: (impairment_facts.resources|length) == 0 105 | delay: 4 106 | # Retry until the operator would finish anyway. 107 | # Set it to never fail because normally it fails after it runs out of retries0 108 | # # The delay is four seconds, plus the second from the operation itself. 109 | retries: "{{ ((sleep_time | int) / 5)|round(0,'ceil')|int }}" 110 | failed_when: False 111 | 112 | - name: Remove Daemonset 113 | k8s: 114 | state: absent 115 | # definition: "{{ lookup('template', 'impairment-daemonset.yaml.j2') | from_yaml }}" 116 | kind: DaemonSet 117 | name: "{{ daemonset_name }}" 118 | namespace: "cluster-impairment-operator" 119 | 120 | - name: Mark as complete 121 | operator_sdk.util.k8s_status: 122 | api_version: apps.redhat.com/v1alpha1 123 | kind: ClusterImpairment 124 | name: "{{ ansible_operator_meta.name }}" 125 | namespace: "{{ ansible_operator_meta.namespace }}" 126 | status: 127 | run_status: "complete" 128 | when: (namespace_facts.resources|length) > 0 and (impairment_facts.resources is defined) and ((impairment_facts.resources|length) > 0) # Resource still exists 129 | 130 | - name: Mark as failed due to missing namespace 131 | operator_sdk.util.k8s_status: 132 | api_version: apps.redhat.com/v1alpha1 133 | kind: ClusterImpairment 134 | name: "{{ ansible_operator_meta.name }}" 135 | namespace: "{{ ansible_operator_meta.namespace }}" 136 | status: 137 | run_status: "failed" 138 | status_details: "Missing namespace cluster-impairment-operator" 139 | when: (namespace_facts.resources|length) == 0 140 | 141 | 142 | 143 | -------------------------------------------------------------------------------- /playbooks/roles/deploy-impairment-workers/templates/impairment-daemonset.yaml.j2: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Red Hat, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: apps/v1 16 | kind: DaemonSet 17 | metadata: 18 | name: "{{ daemonset_name }}" 19 | namespace: cluster-impairment-operator 20 | labels: 21 | k8s-app: "{{ daemonset_name }}" 22 | spec: 23 | selector: 24 | matchLabels: 25 | name: "{{ daemonset_name }}" 26 | template: 27 | metadata: 28 | labels: 29 | name: "{{ daemonset_name }}" 30 | spec: 31 | tolerations: 32 | - key: node-role.kubernetes.io/master 33 | operator: Exists 34 | effect: NoSchedule 35 | {% if node_selector is defined %} 36 | nodeSelector: 37 | "{{ node_selector.key }}": "{{ node_selector.value }}" 38 | {% endif %} # node_selector 39 | containers: 40 | - name: impairment-worker 41 | image: "{{ lookup('env', 'WORKER_IMG') }}" 42 | env: 43 | - name: INTERFACE 44 | value: "{{ impairment_interface_list }}" 45 | 46 | # Ingress 47 | {% if ingress is defined %} 48 | # Ingress bandwidth 49 | {% if ingress.bandwidth is defined %} 50 | - name: INGRESS_BANDWIDTH_LIMIT 51 | value: "{{ ingress.bandwidth }}" 52 | {% endif %} # bandwidth 53 | # Ingress latency 54 | {% if ingress.latency is defined %} 55 | - name: INGRESS_LATENCY 56 | value: "{{ ingress.latency }}" 57 | # Ingress latency_options 58 | {% if ingress.latency_options is defined %} 59 | 60 | {% if ingress.latency_options.jitter is defined %} 61 | # jitter 62 | - name: INGRESS_JITTER 63 | value: "{{ ingress.latency_options.jitter }}" 64 | {% if ingress.latency_options.jitter_correlation is defined %} 65 | # jitterCorrelation 66 | - name: INGRESS_JITTER_CORRELATION 67 | value: "{{ ingress.latency_options.jitter_correlation }}" 68 | {% endif %} # jitter_correlation 69 | {% endif %} # jitter 70 | {% if ingress.latency_options.distribution is defined %} 71 | # distribution 72 | - name: INGRESS_JITTER_DISTRIBUTION 73 | value: "{{ ingress.latency_options.distribution }}" 74 | {% endif %} # distribution 75 | {% if ingress.latency_options.reorder is defined %} 76 | # reorder 77 | - name: INGRESS_REORDER 78 | value: "{{ ingress.latency_options.reorder }}" 79 | {% endif %} # reorder 80 | {% if ingress.latency_options.reorder_correlation is defined %} 81 | # reorder correlation 82 | - name: INGRESS_REORDER_CORRELATION 83 | value: "{{ ingress.latency_options.reorder_correlation }}" 84 | {% endif %} # reorder correlation 85 | {% endif %} # Latency options 86 | {% endif %} # Latency 87 | 88 | # Ingress packet loss 89 | {% if ingress.loss is defined %} 90 | - name: INGRESS_PACKET_LOSS 91 | value: "{{ ingress.loss }}" 92 | # Ingress loss_options 93 | {% if ingress.loss_options is defined %} 94 | # Ingress correlation 95 | {% if ingress.loss_options.correlation is defined %} 96 | - name: INGRESS_LOSS_CORRELATION 97 | value: "{{ ingress.loss_options.correlation }}" 98 | 99 | {% endif %} # correlation 100 | {% endif %} # loss options 101 | {% endif %} # Loss 102 | 103 | 104 | # Ingress corruption 105 | {% if ingress.corruption is defined %} 106 | - name: INGRESS_CORRUPTION 107 | value: "{{ ingress.corruption }}" 108 | # Ingress corelation_options 109 | {% if ingress.corruption_options is defined %} 110 | # Ingress correlation 111 | {% if ingress.corruption_options.correlation is defined %} 112 | - name: INGRESS_CORRUPTION_CORRELATION 113 | value: "{{ ingress.corruption_options.correlation }}" 114 | {% endif %} # correlation 115 | {% endif %} # corruption_options 116 | {% endif %} # Corruption 117 | 118 | # Ingress duplication 119 | {% if ingress.duplication is defined %} 120 | - name: INGRESS_DUPLICATION 121 | value: "{{ ingress.duplication }}" 122 | # Ingress corelation_options 123 | {% if ingress.duplication_options is defined %} 124 | # Ingress correlation 125 | {% if ingress.duplication_options.correlation is defined %} 126 | - name: INGRESS_DUPLICATION_CORRELATION 127 | value: "{{ ingress.duplication_options.correlation }}" 128 | {% endif %} # correlation 129 | {% endif %} # duplication_options 130 | {% endif %} # Duplication 131 | 132 | {% endif %} # Ingress 133 | 134 | # Egress 135 | {% if egress is defined %} 136 | # Egress bandwidth 137 | {% if egress.bandwidth is defined %} 138 | - name: EGRESS_BANDWIDTH_LIMIT 139 | value: "{{ egress.bandwidth }}" 140 | {% endif %} # Bandwidth 141 | # Egress latency 142 | {% if egress.latency is defined %} 143 | - name: EGRESS_LATENCY 144 | value: "{{ egress.latency }}" 145 | 146 | {% if egress.latency_options.jitter is defined %} 147 | # jitter 148 | - name: EGRESS_JITTER 149 | value: "{{ egress.latency_options.jitter }}" 150 | {% if egress.latency_options.jitter_correlation is defined %} 151 | # jitter correlation 152 | - name: EGRESS_JITTER_CORRELATION 153 | value: "{{ egress.latency_options.jitter_correlation }}" 154 | {% endif %} # jitterCorrelation 155 | {% endif %} # jitter 156 | {% if egress.latency_options.distribution is defined %} 157 | # distribution 158 | - name: EGRESS_JITTER_DISTRIBUTION 159 | value: "{{ egress.latency_options.distribution }}" 160 | {% endif %} # distribution 161 | {% if egress.latency_options.reorder is defined %} 162 | # reorder 163 | - name: EGRESS_REORDER 164 | value: "{{ egress.latency_options.reorder }}" 165 | {% endif %} # reorder 166 | {% if egress.latency_options.reorder_correlation is defined %} 167 | # reorder correlation 168 | - name: EGRESS_REORDER_CORRELATION 169 | value: "{{ egress.latency_options.reorder_correlation }}" 170 | {% endif %} # reorderCorrelation 171 | {% endif %} # Latency options 172 | {% endif %} # Latency 173 | 174 | # Egress packet loss 175 | {% if egress.loss is defined %} 176 | - name: EGRESS_PACKET_LOSS 177 | value: "{{ egress.loss }}" 178 | # Egress loss_options 179 | {% if egress.loss_options is defined %} 180 | # Egress correlation 181 | {% if egress.loss_options.correlation is defined %} 182 | - name: EGRESS_LOSS_CORRELATION 183 | value: "{{ egress.loss_options.correlation }}" 184 | 185 | {% endif %} # correlation 186 | {% endif %} # loss options 187 | {% endif %} # loss 188 | 189 | 190 | # Egress corruption 191 | {% if egress.corruption is defined %} 192 | - name: EGRESS_CORRUPTION 193 | value: "{{ egress.corruption }}" 194 | # Egress corelation_options 195 | {% if egress.corruption_options is defined %} 196 | # Egress correlation 197 | {% if egress.corruption_options.correlation is defined %} 198 | - name: EGRESS_CORRUPTION_CORRELATION 199 | value: "{{ egress.corruption_options.correlation }}" 200 | {% endif %} # correlation 201 | {% endif %} # corruption_options 202 | {% endif %} # corruption 203 | 204 | # Egress duplication 205 | {% if egress.duplication is defined %} 206 | - name: EGRESS_DUPLICATION 207 | value: "{{ egress.duplication }}" 208 | # Egress corelation_options 209 | {% if egress.duplication_options is defined %} 210 | # Egress correlation 211 | {% if egress.duplication_options.correlation is defined %} 212 | - name: EGRESS_DUPLICATION_CORRELATION 213 | value: "{{ egress.duplication_options.correlation }}" 214 | {% endif %} # correlation 215 | {% endif %} # duplication_options 216 | {% endif %} # duplication 217 | 218 | # Duration or end time 219 | - name: START_TIME 220 | value: "{{ start_time_epoch }}" 221 | {% if end_time_epoch is defined %} 222 | - name: END_TIME 223 | value: "{{ end_time_epoch }}" 224 | {% else %} 225 | - name: DURATION 226 | value: "-1" 227 | {% endif %} 228 | # Link Flapping Enabled 229 | {% if link_flapping.enable is defined %} 230 | - name: LINK_FLAPPING 231 | value: "{{ link_flapping.enable }}" 232 | {% endif %} 233 | # Link Flapping Down Time 234 | {% if link_flapping.down_time is defined %} 235 | - name: LINK_FLAP_DOWN_TIME 236 | value: "{{ link_flapping.down_time }}" 237 | {% endif %} 238 | # Link Flapping Up Time 239 | {% if link_flapping.up_time is defined %} 240 | - name: LINK_FLAP_UP_TIME 241 | value: "{{ link_flapping.up_time }}" 242 | {% endif %} 243 | # - name: DRY_RUN 244 | # value: "true" 245 | securityContext: 246 | privileged: true 247 | volumeMounts: 248 | - mountPath: /lib/modules 249 | name: modprobe-modules-dir 250 | volumes: 251 | - name: modprobe-modules-dir 252 | hostPath: 253 | path: /lib/modules 254 | hostNetwork: true 255 | dnsPolicy: Default 256 | serviceAccountName: network-access-sa 257 | -------------------------------------------------------------------------------- /playbooks/roles/deploy-impairment-workers/tests/inventory: -------------------------------------------------------------------------------- 1 | localhost 2 | 3 | -------------------------------------------------------------------------------- /playbooks/roles/deploy-impairment-workers/tests/test.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - hosts: localhost 3 | remote_user: root 4 | roles: 5 | - deploy-impairment-workers -------------------------------------------------------------------------------- /playbooks/roles/deploy-impairment-workers/vars/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # vars file for deploy-impairment-workers 3 | -------------------------------------------------------------------------------- /requirements.yml: -------------------------------------------------------------------------------- 1 | --- 2 | collections: 3 | - name: community.kubernetes 4 | version: "1.2.1" 5 | - name: operator_sdk.util 6 | version: "0.2.0" 7 | -------------------------------------------------------------------------------- /watches.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | # Use the 'create api' subcommand to add watches to this file. 3 | - version: v1alpha1 4 | group: apps.redhat.com 5 | kind: ClusterImpairment 6 | playbook: playbooks/clusterimpairment.yml 7 | manageStatus: false 8 | #+kubebuilder:scaffold:watch 9 | --------------------------------------------------------------------------------