├── .ansible-lint ├── .github ├── FUNDING.yml └── workflows │ └── stale.yml ├── .travis.yml ├── .yamllint ├── README.md ├── build ├── Dockerfile ├── chain-operator-files.yml └── test-framework │ ├── Dockerfile │ └── ansible-test.sh ├── deploy ├── crds │ ├── mcrouter_v1alpha3_mcrouter_cr.yaml │ └── mcrouter_v1alpha3_mcrouter_crd.yaml ├── mcrouter-operator.yaml ├── operator.yaml ├── role.yaml ├── role_binding.yaml └── service_account.yaml ├── molecule ├── default │ ├── asserts.yml │ ├── converge.yml │ ├── membash.sh │ ├── molecule.yml │ └── prepare.yml ├── test-cluster │ ├── converge.yml │ └── molecule.yml └── test-local │ ├── converge.yml │ ├── molecule.yml │ └── prepare.yml ├── requirements.yml ├── roles └── mcrouter │ ├── README.md │ ├── defaults │ └── main.yml │ ├── meta │ └── main.yml │ ├── tasks │ └── main.yml │ ├── templates │ ├── mcrouter_deployment.yaml.j2 │ ├── memcache_service.yaml.j2 │ └── memcache_statefulset.yaml.j2 │ └── vars │ └── main.yml └── watches.yaml /.ansible-lint: -------------------------------------------------------------------------------- 1 | skip_list: 2 | - '306' 3 | - '602' 4 | - '503' 5 | 6 | exclude_paths: 7 | - deploy/ 8 | -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | --- 3 | github: geerlingguy 4 | patreon: geerlingguy 5 | -------------------------------------------------------------------------------- /.github/workflows/stale.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: Close inactive issues 3 | 'on': 4 | schedule: 5 | - cron: "55 16 * * 4" # semi-random time 6 | 7 | jobs: 8 | close-issues: 9 | runs-on: ubuntu-latest 10 | permissions: 11 | issues: write 12 | pull-requests: write 13 | steps: 14 | - uses: actions/stale@v8 15 | with: 16 | days-before-stale: 120 17 | days-before-close: 60 18 | exempt-issue-labels: bug,pinned,security,planned 19 | exempt-pr-labels: bug,pinned,security,planned 20 | stale-issue-label: "stale" 21 | stale-pr-label: "stale" 22 | stale-issue-message: | 23 | This issue has been marked 'stale' due to lack of recent activity. If there is no further activity, the issue will be closed in another 30 days. Thank you for your contribution! 24 | 25 | Please read [this blog post](https://www.jeffgeerling.com/blog/2020/enabling-stale-issue-bot-on-my-github-repositories) to see the reasons why I mark issues as stale. 26 | close-issue-message: | 27 | This issue has been closed due to inactivity. If you feel this is in error, please reopen the issue or file a new issue with the relevant details. 28 | stale-pr-message: | 29 | This pr has been marked 'stale' due to lack of recent activity. If there is no further activity, the issue will be closed in another 30 days. Thank you for your contribution! 30 | 31 | Please read [this blog post](https://www.jeffgeerling.com/blog/2020/enabling-stale-issue-bot-on-my-github-repositories) to see the reasons why I mark issues as stale. 32 | close-pr-message: | 33 | This pr has been closed due to inactivity. If you feel this is in error, please reopen the issue or file a new issue with the relevant details. 34 | repo-token: ${{ secrets.GITHUB_TOKEN }} 35 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | --- 2 | sudo: required 3 | services: docker 4 | language: python 5 | 6 | before_install: 7 | # Upgrade Docker to work with docker-py. 8 | - curl https://gist.githubusercontent.com/geerlingguy/ce883ad4aec6a5f1187ef93bd338511e/raw/36612d28981d92863f839c5aefe5b7dd7193d6c6/travis-ci-docker-upgrade.sh | sudo bash 9 | 10 | install: 11 | - pip install ansible molecule[docker] yamllint ansible-lint docker openshift jmespath 12 | 13 | script: 14 | - molecule test -s test-local 15 | -------------------------------------------------------------------------------- /.yamllint: -------------------------------------------------------------------------------- 1 | --- 2 | extends: default 3 | 4 | rules: 5 | braces: 6 | max-spaces-inside: 1 7 | level: error 8 | brackets: 9 | max-spaces-inside: 1 10 | level: error 11 | line-length: disable 12 | truthy: disable 13 | 14 | ignore: | 15 | .github/workflows/stale.yml 16 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Mcrouter Operator 2 | 3 | [![Build Status](https://travis-ci.com/geerlingguy/mcrouter-operator.svg?branch=master)](https://travis-ci.com/geerlingguy/mcrouter-operator) 4 | 5 | The Mcrouter Operator was built with the [Ansible Operator SDK](https://github.com/operator-framework/operator-sdk/blob/master/doc/ansible/user-guide.md). It is not yet intended for production use. 6 | 7 | [Mcrouter](https://github.com/facebook/mcrouter) is a memcached protocol router for scaling memcached deployments, written by Facebook. 8 | 9 | [Dylan Murray](https://github.com/dymurray)'s memcached operator was the original inspiration for this operator, and this project was originally forked from the [AnsOpDemo](https://github.com/Ansible-Getting-Started/AnsOpDemo) repository. 10 | 11 | ## Usage 12 | 13 | This Kubernetes Operator is meant to be deployed in your Kubernetes cluster(s) and can manage one or more mcrouter instances in the same namespace as the operator. 14 | 15 | First you need to deploy Mcrouter Operator into your cluster: 16 | 17 | kubectl apply -f https://raw.githubusercontent.com/geerlingguy/mcrouter-operator/master/deploy/mcrouter-operator.yaml 18 | 19 | Then you can create instances of mcrouter, for example: 20 | 21 | 1. Create a file named `my-mcrouter.yml` with the following contents: 22 | 23 | ``` 24 | --- 25 | apiVersion: mcrouter.example.com/v1alpha3 26 | kind: Mcrouter 27 | metadata: 28 | name: my-mcrouter 29 | spec: 30 | memcached_pool_size: 3 31 | # The memcached pool can be 'sharded' or 'replicated'. 32 | pool_setup: replicated 33 | ``` 34 | 35 | 2. Use `kubectl` to create the mcrouter instance in your cluster: 36 | 37 | ``` 38 | kubectl apply -f my-mcrouter.yml 39 | ``` 40 | 41 | > **What's the difference between `sharded` and `replicated`**: `sharded` uses a key hashing algorithm to distribute Memcached `set`s and `get`s among Memcached Pods; this means a key `foo` may always go to pod A, while the key `bar` always goes to pod B. `replicated` sends all Memcached `set`s to all Memcached pods, and distributes `get`s randomly. 42 | 43 | ### Available parameters in the Mcrouter spec 44 | 45 | There are a number of configurable options in the `spec` for your Mcrouter resources. For full details on each available variable, see the [mcrouter role README](roles/mcrouter/README.md). 46 | 47 | spec: 48 | # The image to run for the `mcrouter` Deployment. 49 | mcrouter_image: devan2502/mcrouter:latest 50 | 51 | # The port Mcrouter will run on. 52 | mcrouter_port: 5000 53 | 54 | # The image to run for the `memcached` StatefulSet. 55 | memcached_image: memcached:1.5-alpine 56 | 57 | # The size of the memcached pool. 58 | memcached_pool_size: 3 59 | 60 | # The port Memcached will run on. 61 | memcached_port: 11211 62 | 63 | # The memcached pool can be 'sharded' or 'replicated'. 64 | pool_setup: replicated 65 | 66 | # Set to '/var/mcrouter/fifos' to debug mcrouter with mcpiper. 67 | debug_fifo_root: '' 68 | 69 | ## Development 70 | 71 | ### Testing 72 | 73 | #### Local tests with Molecule and KIND 74 | 75 | Ensure you have the testing dependencies installed (in addition to Docker): 76 | 77 | pip install docker molecule openshift jmespath 78 | 79 | Run the local molecule test scenario: 80 | 81 | molecule test -s test-local 82 | 83 | #### Testing if mcrouter and memcached are working as expected 84 | 85 | Get the Kubernetes network IP address for the mcrouter pod: 86 | 87 | kubectl describe pod -l app=mcrouter 88 | 89 | Then run a `telnet` container to connect to mcrouter directly: 90 | 91 | ```sh 92 | kubectl run -it --rm telnet --image=jess/telnet --restart=Never 5000 93 | ``` 94 | 95 | After a few seconds you will see a message like `If you don't see a command prompt, try pressing enter.`. Don't press enter, because telnet doesn't display a prompt. Instead, enter the below commands: 96 | 97 | In the telnet prompt send commands like the following: 98 | 99 | ``` 100 | set mykey 0 0 5 101 | hello 102 | get mykey 103 | stats 104 | quit 105 | ``` 106 | 107 | You can also inspect Mcrouter fifos using `mcpiper`, by setting `spec.debug_fifo_root` to `/var/mcrouter/fifos`, then running `mcpiper` inside the mcrouter pod once it's reconfigured: `/usr/local/mcrouter/install/bin/mcpiper`. Note that you will not see any output (besides maybe an error message) until requests are sent to mcrouter. 108 | 109 | #### Testing the operator is working as expected 110 | 111 | One simple way to verify Mcrouter operator is working correctly is to change the `memcached_pool_size` in your Mcrouter resource, then observe what happens in the cluster: 112 | 113 | 1. See how many memcached pods are currently running: `kubectl get pods -l app=mcrouter-cache` 114 | 1. See how Mcrouter is currently configured: `kubectl describe pod -l app=mcrouter` 115 | 1. Verify all the current memcached pods are listed in the 'servers' inside `--config-str` in the mcrouter container command. 116 | 1. Edit the Mcrouter resource: `kubectl edit mcrouter my-mcrouter` 117 | 1. Change `memcached_pool_size` to `4` 118 | 1. Save the change. 119 | 1. Check the status of your mcrouter instance: `kubectl describe mcrouter my-mcrouter` 120 | 1. For a minute or two, the operator will be running a 'reconciliation' to ensure the cluster is updated to reflect the updated mcrouter spec you just saved. 121 | 1. Once it's done applying the necessary changes, the status message will read "Awaiting next reconciliation". 122 | 1. See how many memcached pods are now running: `kubectl get pods -l app=mcrouter-cache` 123 | 1. You should now see four pods in the list. 124 | 1. See how Mcrouter is now configured: `kubectl describe pod -l app=mcrouter` 125 | 1. You should now see all four of the memcached pods in the `--config-str`. 126 | 127 | Another thing you could do is delete one of the Memcached pods and see what happens: 128 | 129 | kubectl delete pod mcrouter-memcached-2 130 | 131 | Within a few seconds, you should see that the deleted pod has been replaced by Kubernetes. Because mcrouter uses DNS to distribute requests to the memcached instances, there might be a single dropped connection if there was an active request to the deleted instance, but other than that rare occurrence, the loss of a single memcached pod should have no affect on the availability of the entire cache backend. 132 | 133 | Note that because Memcached does not include any kind of replication, and Mcrouter does not backfill new Memcached instances (they will start with an empty cache), advanced usage of Mcrouter and Memcached in high-availability, high-performance environments requires more fine tuning in the replication strategy. 134 | 135 | Also, your application should always be able to fall back and re-set a key if a particular cache key has vanished. Basically, don't rely on Mcrouter or Memcached for a data persistence layer! 136 | 137 | ### Release Process 138 | 139 | There are a few moving parts to this project: 140 | 141 | 1. The Docker image which powers Mcrouter Operator. 142 | 2. The `mcrouter-operator.yaml` Kubernetes manifest file which initially deploys the Operator into a cluster. 143 | 144 | Each of these must be appropriately built in preparation for a new tag: 145 | 146 | #### Build a new release of the Operator for Docker Hub 147 | 148 | Run the following command inside this directory: 149 | 150 | operator-sdk build geerlingguy/mcrouter-operator:0.2.2 151 | 152 | Then push the generated image to Docker Hub: 153 | 154 | docker push geerlingguy/mcrouter-operator:0.2.2 155 | 156 | #### Build a new version of the `mcrouter-operator.yaml` file 157 | 158 | Update the mcrouter-operator version in two places: 159 | 160 | 1. `deploy/mcrouter-operator.yaml`: in the `ansible` and `operator` container definitions in the `mcrouter-operator` Deployment. 161 | 2. `build/chain-operator-files.yml`: the `operator_image` variable. 162 | 163 | Once the versions are updated, run the playbook in the `build/` directory: 164 | 165 | ansible-playbook chain-operator-files.yml 166 | 167 | After it is built, test it on a local cluster: 168 | 169 | minikube start 170 | kubectl apply -f deploy/mcrouter-operator.yaml 171 | kubectl apply -f deploy/crds/mcrouter_v1alpha3_mcrouter_cr.yaml 172 | 173 | minikube delete 174 | 175 | If everything works, commit the updated version, then tag a new repository release with the same tag as the Docker image pushed earlier. 176 | 177 | ## More resources for Ansible Operator SDK and Mcrouter 178 | 179 | - [Mcrouter Operator - demonstration of K8s Operator SDK usage with Ansible](https://www.jeffgeerling.com/blog/2019/mcrouter-operator-demonstration-k8s-operator-sdk-usage-ansible) 180 | - [Ansible Operator: What is it? Why it matters? What can you do with it?](https://www.ansible.com/blog/ansible-operator) 181 | - [An introduction to Ansible Operators in Kubernetes](https://opensource.com/article/18/10/ansible-operators-kubernetes) 182 | - [Reaching for the Stars with Ansible Operator](https://blog.openshift.com/reaching-for-the-stars-with-ansible-operator/) 183 | - [Mcrouter Wiki](https://github.com/facebook/mcrouter/wiki) 184 | - [Mcrouter Docker image](https://github.com/Dev25/mcrouter-docker/) 185 | - [A Practical kubernetes Operator using Ansible — an example](https://itnext.io/a-practical-kubernetes-operator-using-ansible-an-example-d3a9d3674d5b) 186 | - [Mcrouter Helm Chart](https://github.com/helm/charts/tree/master/stable/mcrouter) 187 | -------------------------------------------------------------------------------- /build/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM quay.io/operator-framework/ansible-operator:v0.16.0 2 | 3 | COPY requirements.yml ${HOME}/requirements.yml 4 | RUN ansible-galaxy collection install -r ${HOME}/requirements.yml \ 5 | && chmod -R ug+rwx ${HOME}/.ansible 6 | 7 | COPY watches.yaml ${HOME}/watches.yaml 8 | 9 | COPY roles/ ${HOME}/roles/ 10 | 11 | -------------------------------------------------------------------------------- /build/chain-operator-files.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # To run: `ansible-playbook chain-operator-files.yml` 3 | - name: Chain operator files together for easy deployment. 4 | hosts: localhost 5 | connection: local 6 | gather_facts: false 7 | 8 | vars: 9 | operator_image: geerlingguy/mcrouter-operator:0.2.2 10 | pull_policy: Always 11 | operator_file_path: "../deploy/mcrouter-operator.yaml" 12 | operator_template: "../deploy/operator.yaml" 13 | 14 | tasks: 15 | - name: Clear out current contents of mcrouter-operator.yml 16 | copy: 17 | dest: "{{ operator_file_path }}" 18 | content: '' 19 | force: true 20 | 21 | - name: Concatenate operator files into mcrouter-operator.yml 22 | blockinfile: 23 | path: "{{ operator_file_path }}" 24 | block: "{{ item }}" 25 | marker: "" 26 | marker_begin: "" 27 | marker_end: "" 28 | insertafter: "EOF" 29 | with_file: 30 | - "../deploy/crds/mcrouter_v1alpha3_mcrouter_crd.yaml" 31 | - "../deploy/role.yaml" 32 | - "../deploy/role_binding.yaml" 33 | - "../deploy/service_account.yaml" 34 | - "../deploy/operator.yaml" 35 | 36 | - name: Remove space at beginning of mcrouter-operator.yml 37 | shell: > 38 | echo "$(tail -n +2 {{ operator_file_path }})" > {{ operator_file_path }} 39 | changed_when: true 40 | 41 | - name: Template the mcrouter-operator.yaml file into mcrouter-operator.yaml 42 | template: 43 | src: "{{ operator_file_path }}" 44 | dest: "{{ operator_file_path }}" 45 | -------------------------------------------------------------------------------- /build/test-framework/Dockerfile: -------------------------------------------------------------------------------- 1 | ARG BASEIMAGE 2 | FROM ${BASEIMAGE} 3 | USER 0 4 | 5 | RUN yum install -y python-devel gcc libffi-devel 6 | RUN pip install molecule==2.20.1 7 | 8 | ARG NAMESPACEDMAN 9 | ADD $NAMESPACEDMAN /namespaced.yaml 10 | ADD build/test-framework/ansible-test.sh /ansible-test.sh 11 | RUN chmod +x /ansible-test.sh 12 | USER 1001 13 | ADD . /opt/ansible/project 14 | -------------------------------------------------------------------------------- /build/test-framework/ansible-test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | export WATCH_NAMESPACE=${TEST_NAMESPACE} 3 | (/usr/local/bin/entrypoint)& 4 | trap "kill $!" SIGINT SIGTERM EXIT 5 | 6 | cd ${HOME}/project 7 | exec molecule test -s test-cluster 8 | -------------------------------------------------------------------------------- /deploy/crds/mcrouter_v1alpha3_mcrouter_cr.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: mcrouter.example.com/v1alpha3 3 | kind: Mcrouter 4 | metadata: 5 | name: mcrouter 6 | spec: 7 | mcrouter_image: devan2502/mcrouter:latest 8 | mcrouter_port: 5000 9 | memcached_image: memcached:1.5-alpine 10 | memcached_pool_size: 2 11 | memcached_port: 11211 12 | pool_setup: replicated # or 'sharded' 13 | debug_fifo_root: '' # set to '/var/mcrouter/fifos' for mcpiper 14 | -------------------------------------------------------------------------------- /deploy/crds/mcrouter_v1alpha3_mcrouter_crd.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apiextensions.k8s.io/v1beta1 3 | kind: CustomResourceDefinition 4 | metadata: 5 | name: mcrouters.mcrouter.example.com 6 | spec: 7 | group: mcrouter.example.com 8 | names: 9 | kind: Mcrouter 10 | listKind: McrouterList 11 | plural: mcrouters 12 | singular: mcrouter 13 | scope: Namespaced 14 | subresources: 15 | status: {} 16 | version: v1alpha3 17 | versions: 18 | - name: v1alpha3 19 | served: true 20 | storage: true 21 | -------------------------------------------------------------------------------- /deploy/mcrouter-operator.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: Role 4 | metadata: 5 | creationTimestamp: null 6 | name: mcrouter-operator 7 | rules: 8 | - apiGroups: 9 | - "" 10 | resources: 11 | - pods 12 | - services 13 | - endpoints 14 | - persistentvolumeclaims 15 | - events 16 | - configmaps 17 | - secrets 18 | verbs: 19 | - '*' 20 | - apiGroups: 21 | - apps 22 | resources: 23 | - deployments 24 | - daemonsets 25 | - replicasets 26 | - statefulsets 27 | verbs: 28 | - '*' 29 | - apiGroups: 30 | - monitoring.coreos.com 31 | resources: 32 | - servicemonitors 33 | verbs: 34 | - get 35 | - create 36 | - apiGroups: 37 | - apps 38 | resourceNames: 39 | - mcrouter-operator 40 | resources: 41 | - deployments/finalizers 42 | verbs: 43 | - update 44 | - apiGroups: 45 | - memcached.example.com 46 | resources: 47 | - '*' 48 | verbs: 49 | - '*' 50 | - apiGroups: 51 | - mcrouter.example.com 52 | resources: 53 | - '*' 54 | - mcrouters 55 | verbs: 56 | - '*' 57 | 58 | --- 59 | kind: RoleBinding 60 | apiVersion: rbac.authorization.k8s.io/v1 61 | metadata: 62 | name: mcrouter-operator 63 | subjects: 64 | - kind: ServiceAccount 65 | name: mcrouter-operator 66 | roleRef: 67 | kind: Role 68 | name: mcrouter-operator 69 | apiGroup: rbac.authorization.k8s.io 70 | 71 | --- 72 | apiVersion: v1 73 | kind: ServiceAccount 74 | metadata: 75 | name: mcrouter-operator 76 | 77 | --- 78 | apiVersion: apps/v1 79 | kind: Deployment 80 | metadata: 81 | name: mcrouter-operator 82 | spec: 83 | replicas: 1 84 | selector: 85 | matchLabels: 86 | name: mcrouter-operator 87 | template: 88 | metadata: 89 | labels: 90 | name: mcrouter-operator 91 | spec: 92 | serviceAccountName: mcrouter-operator 93 | containers: 94 | - name: ansible 95 | command: 96 | - /usr/local/bin/ao-logs 97 | - /tmp/ansible-operator/runner 98 | - stdout 99 | image: "geerlingguy/mcrouter-operator:0.2.2" 100 | imagePullPolicy: "Always" 101 | volumeMounts: 102 | - mountPath: /tmp/ansible-operator/runner 103 | name: runner 104 | readOnly: true 105 | - name: operator 106 | image: "geerlingguy/mcrouter-operator:0.2.2" 107 | imagePullPolicy: "Always" 108 | volumeMounts: 109 | - mountPath: /tmp/ansible-operator/runner 110 | name: runner 111 | env: 112 | - name: WATCH_NAMESPACE 113 | valueFrom: 114 | fieldRef: 115 | fieldPath: metadata.namespace 116 | - name: POD_NAME 117 | valueFrom: 118 | fieldRef: 119 | fieldPath: metadata.name 120 | - name: OPERATOR_NAME 121 | value: "mcrouter-operator" 122 | volumes: 123 | - name: runner 124 | emptyDir: {} 125 | 126 | --- 127 | apiVersion: apiextensions.k8s.io/v1beta1 128 | kind: CustomResourceDefinition 129 | metadata: 130 | name: mcrouters.mcrouter.example.com 131 | spec: 132 | group: mcrouter.example.com 133 | names: 134 | kind: Mcrouter 135 | listKind: McrouterList 136 | plural: mcrouters 137 | singular: mcrouter 138 | scope: Namespaced 139 | subresources: 140 | status: {} 141 | version: v1alpha3 142 | versions: 143 | - name: v1alpha3 144 | served: true 145 | storage: true 146 | -------------------------------------------------------------------------------- /deploy/operator.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apps/v1 3 | kind: Deployment 4 | metadata: 5 | name: mcrouter-operator 6 | spec: 7 | replicas: 1 8 | selector: 9 | matchLabels: 10 | name: mcrouter-operator 11 | template: 12 | metadata: 13 | labels: 14 | name: mcrouter-operator 15 | spec: 16 | serviceAccountName: mcrouter-operator 17 | containers: 18 | - name: ansible 19 | command: 20 | - /usr/local/bin/ao-logs 21 | - /tmp/ansible-operator/runner 22 | - stdout 23 | image: "{{ operator_image }}" 24 | imagePullPolicy: "{{ pull_policy }}" 25 | volumeMounts: 26 | - mountPath: /tmp/ansible-operator/runner 27 | name: runner 28 | readOnly: true 29 | - name: operator 30 | image: "{{ operator_image }}" 31 | imagePullPolicy: "{{ pull_policy }}" 32 | volumeMounts: 33 | - mountPath: /tmp/ansible-operator/runner 34 | name: runner 35 | env: 36 | - name: WATCH_NAMESPACE 37 | valueFrom: 38 | fieldRef: 39 | fieldPath: metadata.namespace 40 | - name: POD_NAME 41 | valueFrom: 42 | fieldRef: 43 | fieldPath: metadata.name 44 | - name: OPERATOR_NAME 45 | value: "mcrouter-operator" 46 | volumes: 47 | - name: runner 48 | emptyDir: {} 49 | -------------------------------------------------------------------------------- /deploy/role.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: Role 4 | metadata: 5 | creationTimestamp: null 6 | name: mcrouter-operator 7 | rules: 8 | - apiGroups: 9 | - "" 10 | resources: 11 | - pods 12 | - services 13 | - endpoints 14 | - persistentvolumeclaims 15 | - events 16 | - configmaps 17 | - secrets 18 | verbs: 19 | - '*' 20 | - apiGroups: 21 | - apps 22 | resources: 23 | - deployments 24 | - daemonsets 25 | - replicasets 26 | - statefulsets 27 | verbs: 28 | - '*' 29 | - apiGroups: 30 | - monitoring.coreos.com 31 | resources: 32 | - servicemonitors 33 | verbs: 34 | - get 35 | - create 36 | - apiGroups: 37 | - apps 38 | resourceNames: 39 | - mcrouter-operator 40 | resources: 41 | - deployments/finalizers 42 | verbs: 43 | - update 44 | - apiGroups: 45 | - memcached.example.com 46 | resources: 47 | - '*' 48 | verbs: 49 | - '*' 50 | - apiGroups: 51 | - mcrouter.example.com 52 | resources: 53 | - '*' 54 | - mcrouters 55 | verbs: 56 | - '*' 57 | -------------------------------------------------------------------------------- /deploy/role_binding.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | kind: RoleBinding 3 | apiVersion: rbac.authorization.k8s.io/v1 4 | metadata: 5 | name: mcrouter-operator 6 | subjects: 7 | - kind: ServiceAccount 8 | name: mcrouter-operator 9 | roleRef: 10 | kind: Role 11 | name: mcrouter-operator 12 | apiGroup: rbac.authorization.k8s.io 13 | -------------------------------------------------------------------------------- /deploy/service_account.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: ServiceAccount 4 | metadata: 5 | name: mcrouter-operator 6 | -------------------------------------------------------------------------------- /molecule/default/asserts.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Verify 3 | hosts: localhost 4 | connection: local 5 | 6 | vars: 7 | ansible_python_interpreter: '{{ ansible_playbook_python }}' 8 | test_keys: 9 | - key: foo 10 | value: bar 11 | - key: fizz 12 | value: buzz 13 | 14 | tasks: 15 | - name: Get mcrouter Pod data 16 | k8s_facts: 17 | kind: Pod 18 | namespace: '{{ namespace }}' 19 | label_selectors: 20 | - app=mcrouter 21 | register: mcrouter_pod 22 | 23 | - name: Set variable with mcrouter IP 24 | set_fact: 25 | mcrouter_pod_ip: "{{ mcrouter_pod.resources[0].status.podIP }}" 26 | 27 | - name: Test setting keys via mcrouter 28 | script: ./membash.sh -h {{ mcrouter_pod_ip }} -p 5000 set {{ item.key }} 0 {{ item.value }} 29 | register: membash_key 30 | delegate_to: "{{ groups.k8s[0] }}" 31 | with_items: "{{ test_keys }}" 32 | failed_when: membash_key.stdout_lines[0] != 'STORED' 33 | 34 | - name: Test getting keys from mcrouter 35 | script: ./membash.sh -h {{ mcrouter_pod_ip }} -p 5000 get {{ item.key }} 36 | register: membash_key 37 | delegate_to: "{{ groups.k8s[0] }}" 38 | with_items: "{{ test_keys }}" 39 | failed_when: membash_key.stdout_lines[0] != item.value 40 | 41 | - name: Ensure there are 2 servers up according to mcrouter 42 | script: ./membash.sh -h {{ mcrouter_pod_ip }} -p 5000 stats 43 | register: membash_stats 44 | delegate_to: "{{ groups.k8s[0] }}" 45 | failed_when: "'STAT num_servers_up 2' not in membash_stats.stdout" 46 | 47 | - name: Get memcached Pod data 48 | k8s_facts: 49 | kind: Pod 50 | namespace: '{{ namespace }}' 51 | label_selectors: 52 | - app=mcrouter-cache 53 | register: memcached_pods 54 | 55 | - name: Test getting keys directly from memcached pod 0 with IP {{ memcached_pods.resources[0].status.podIP }} 56 | script: ./membash.sh -h {{ memcached_pods.resources[0].status.podIP }} get {{ item.key }} 57 | register: membash_key 58 | delegate_to: "{{ groups.k8s[0] }}" 59 | with_items: "{{ test_keys }}" 60 | failed_when: membash_key.stdout_lines[0] != item.value 61 | 62 | - name: Test getting keys directly from memcached pod 1 with IP {{ memcached_pods.resources[1].status.podIP }} 63 | script: ./membash.sh -h {{ memcached_pods.resources[1].status.podIP }} get {{ item.key }} 64 | register: membash_key 65 | delegate_to: "{{ groups.k8s[0] }}" 66 | with_items: "{{ test_keys }}" 67 | failed_when: membash_key.stdout_lines[0] != item.value 68 | -------------------------------------------------------------------------------- /molecule/default/converge.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Converge 3 | hosts: localhost 4 | connection: local 5 | vars: 6 | ansible_python_interpreter: '{{ ansible_playbook_python }}' 7 | roles: 8 | - mcrouter 9 | 10 | - import_playbook: '{{ playbook_dir }}/asserts.yml' 11 | -------------------------------------------------------------------------------- /molecule/default/membash.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Gist: 11375877 4 | # Url: https://gist.github.com/goodevilgenius/11375877 5 | # 6 | # All memcache functions are supported. 7 | # 8 | # Can also be sourced from other scripts, e.g. 9 | # source membash.sh 10 | # MCSERVER="localhost" 11 | # MCPORT=11211 12 | # foobar=$(mc_get foobar) 13 | # [ -z "$foobar" ] && foobar="default value" 14 | # mc_set foobar 0 "$foobar" 15 | 16 | # original author: wumin, https://gist.github.com/ri0day/1538831 17 | # updated by goodevilgenius to support debian-based systems, support more 18 | # functions, and be more user-friendly 19 | 20 | mc_usage() { 21 | format_usage="membash: a memcache library for BASH \n\ 22 | https://gist.github.com/goodevilgenius/11375877\n\n\ 23 | Usage:\n 24 | \t $(basename "$0") [-hp] command [arguments] \n \ 25 | \t [-h]\t memcached hostname or ip. \n \ 26 | \t [-p]\t memcached port. \n\n\ 27 | Commands: \n \ 28 | \t usage (print this help) \n \ 29 | \t set/add/replace/append/prepend key exptime value \n \ 30 | \t touch key exptime \n \ 31 | \t incr/decr key value \n \ 32 | \t get key \n \ 33 | \t delete key [time] \n \ 34 | \t stats \n \ 35 | \t list_all_keys" 36 | echo -e $format_usage 37 | } 38 | mc_help() { mc_usage;} 39 | 40 | mc_sendmsg() { echo -e "$*\r" | nc $MCSERVER $MCPORT;} 41 | 42 | mc_stats() { mc_sendmsg "stats";} 43 | 44 | mc_get_last_items_id() { 45 | LastID=$(mc_sendmsg "stats items"|tail -n 2|head -n 1|awk -F':' '{print $2}') 46 | echo $LastID 47 | } 48 | 49 | mc_list_all_keys() { 50 | :>/dev/shm/mc_all_keys_${MCSERVER}_${MCPORT}.txt 51 | max_item_num=$(mc_get_last_items_id) 52 | for i in `seq 1 $max_item_num` 53 | do 54 | mc_sendmsg "stats cachedump $i 0" | awk '{print $2}' 55 | done >>/dev/shm/mc_all_keys_${MCSERVER}_${MCPORT}.txt 56 | sed -i '/^$/d' /dev/shm/mc_all_keys_${MCSERVER}_${MCPORT}.txt 57 | cat /dev/shm/mc_all_keys_${MCSERVER}_${MCPORT}.txt 58 | } 59 | 60 | mc_get() { mc_sendmsg "get $1" | awk "/^VALUE $1/{a=1;next}/^END/{a=0}a" ;} 61 | 62 | mc_touch() { 63 | key="$1" 64 | shift 65 | let exptime="$1" 66 | shift 67 | mc_sendmsg "touch $key $exptime" 68 | } 69 | 70 | mc_doset() { 71 | command="$1" 72 | shift 73 | key="$1" 74 | shift 75 | let exptime="$1" 76 | shift 77 | val="$*" 78 | let bytes=$(echo -n "$val"|wc -c) 79 | mc_sendmsg "$command $key 0 $exptime $bytes\r\n$val" 80 | } 81 | 82 | mc_set() { mc_doset set "$@";} 83 | mc_add() { mc_doset add "$@";} 84 | mc_replace() { mc_doset replace "$@";} 85 | mc_append() { mc_doset append "$@";} 86 | mc_prepend() { mc_doset prepend "$@";} 87 | 88 | mc_delete() { mc_sendmsg delete "$*";} 89 | mc_incr() { mc_sendmsg incr "$*";} 90 | mc_decr() { mc_sendmsg decr "$*";} 91 | 92 | mc_superpurge() { 93 | mc_list_all_keys > /dev/null 94 | if [ ! -z "/dev/shm/mc_all_keys_${MCSERVER}_${MCPORT}.txt" ];then 95 | grep "$1" /dev/shm/mc_all_keys_${MCSERVER}_${MCPORT}.txt >/dev/shm/temp.swap.${MCSERVER}_${MCPORT}.txt 96 | fi 97 | while read keys 98 | do 99 | mc_sendmsg "delete ${keys}" 100 | done &2 122 | ;; 123 | esac 124 | done 125 | command="${@:$OPTIND:1}" 126 | [ -z "$command" ] && command="usage" 127 | let OPTIND++ 128 | 129 | mc_$command "${@:$OPTIND}" 130 | 131 | exit $? 132 | 133 | fi -------------------------------------------------------------------------------- /molecule/default/molecule.yml: -------------------------------------------------------------------------------- 1 | --- 2 | dependency: 3 | name: galaxy 4 | driver: 5 | name: docker 6 | lint: | 7 | set -e 8 | yamllint . 9 | ansible-lint 10 | platforms: 11 | - name: kind-default 12 | groups: 13 | - k8s 14 | image: bsycorp/kind:latest-1.15 15 | privileged: True 16 | override_command: no 17 | exposed_ports: 18 | - 8443/tcp 19 | - 10080/tcp 20 | published_ports: 21 | - 0.0.0.0:${TEST_CLUSTER_PORT:-9443}:8443/tcp 22 | pre_build_image: yes 23 | provisioner: 24 | name: ansible 25 | log: True 26 | inventory: 27 | group_vars: 28 | all: 29 | namespace: ${TEST_NAMESPACE:-default} 30 | env: 31 | K8S_AUTH_KUBECONFIG: /tmp/molecule/kind-default/kubeconfig 32 | KUBECONFIG: /tmp/molecule/kind-default/kubeconfig 33 | ANSIBLE_ROLES_PATH: ${MOLECULE_PROJECT_DIRECTORY}/roles 34 | KIND_PORT: '${TEST_CLUSTER_PORT:-9443}' 35 | -------------------------------------------------------------------------------- /molecule/default/prepare.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Prepare 3 | hosts: k8s 4 | gather_facts: no 5 | vars: 6 | kubeconfig: "{{ lookup('env', 'KUBECONFIG') }}" 7 | tasks: 8 | - name: delete the kubeconfig if present 9 | file: 10 | path: '{{ kubeconfig }}' 11 | state: absent 12 | delegate_to: localhost 13 | 14 | - name: Fetch the kubeconfig 15 | fetch: 16 | dest: '{{ kubeconfig }}' 17 | flat: yes 18 | src: /root/.kube/config 19 | 20 | - name: Change the kubeconfig port to the proper value 21 | replace: 22 | regexp: "8443" 23 | replace: "{{ lookup('env', 'KIND_PORT') }}" 24 | path: '{{ kubeconfig }}' 25 | mode: 0644 26 | delegate_to: localhost 27 | 28 | - name: Wait for the Kubernetes API to become available (this could take a minute) 29 | uri: 30 | url: "http://localhost:10080/kubernetes-ready" 31 | status_code: 200 32 | validate_certs: no 33 | register: result 34 | until: (result.status|default(-1)) == 200 35 | retries: 60 36 | delay: 5 37 | -------------------------------------------------------------------------------- /molecule/test-cluster/converge.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Converge 3 | hosts: localhost 4 | connection: local 5 | 6 | vars: 7 | ansible_python_interpreter: '{{ ansible_playbook_python }}' 8 | deploy_dir: "{{ lookup('env', 'MOLECULE_PROJECT_DIRECTORY') }}/deploy" 9 | image_name: mcrouter.example.com/mcrouter-operator:testing 10 | custom_resource: "{{ lookup('file', '/'.join([deploy_dir, 'crds/mcrouter_v1alpha3_mcrouter_cr.yaml'])) | from_yaml }}" 11 | 12 | tasks: 13 | - name: Create the mcrouter.example.com/v1alpha1.Mcrouter 14 | k8s: 15 | namespace: '{{ namespace }}' 16 | definition: "{{ lookup('file', '/'.join([deploy_dir, 'crds/mcrouter_v1alpha3_mcrouter_cr.yaml'])) }}" 17 | 18 | - name: Get the newly created Custom Resource 19 | debug: 20 | msg: "{{ lookup('k8s', 21 | group='mcrouter.example.com', 22 | api_version='v1alpha3', 23 | kind='Mcrouter', 24 | namespace=namespace, 25 | resource_name=custom_resource.metadata.name 26 | )}}" 27 | 28 | - name: Wait 5m for reconciliation to run 29 | k8s_facts: 30 | api_version: 'v1alpha3' 31 | kind: 'Mcrouter' 32 | namespace: '{{ namespace }}' 33 | name: '{{ custom_resource.metadata.name }}' 34 | register: reconcile_cr 35 | until: 36 | - "'Successful' in (reconcile_cr | json_query('resources[].status.conditions[].reason'))" 37 | delay: 5 38 | retries: 60 39 | 40 | - import_playbook: '{{ playbook_dir }}/../default/asserts.yml' 41 | -------------------------------------------------------------------------------- /molecule/test-cluster/molecule.yml: -------------------------------------------------------------------------------- 1 | --- 2 | dependency: 3 | name: galaxy 4 | driver: 5 | name: delegated 6 | options: 7 | managed: False 8 | ansible_connection_options: {} 9 | lint: | 10 | set -e 11 | yamllint . 12 | ansible-lint 13 | platforms: 14 | - name: test-cluster 15 | groups: 16 | - k8s 17 | provisioner: 18 | name: ansible 19 | inventory: 20 | group_vars: 21 | all: 22 | namespace: ${TEST_NAMESPACE:-osdk-test} 23 | env: 24 | ANSIBLE_ROLES_PATH: ${MOLECULE_PROJECT_DIRECTORY}/roles 25 | scenario: 26 | test_sequence: 27 | - lint 28 | - destroy 29 | - dependency 30 | - syntax 31 | - create 32 | - prepare 33 | - converge 34 | - side_effect 35 | - verify 36 | - destroy 37 | -------------------------------------------------------------------------------- /molecule/test-local/converge.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Build Operator in Kubernetes docker container 3 | hosts: k8s 4 | gather_facts: false 5 | 6 | vars: 7 | image_name: mcrouter.example.com/mcrouter-operator:testing 8 | 9 | tasks: 10 | # using command so we don't need to install any dependencies 11 | - name: Get existing image hash 12 | command: docker images -q {{ image_name }} 13 | register: prev_hash 14 | changed_when: false 15 | 16 | - name: Build Operator Image 17 | command: docker build -f /build/build/Dockerfile -t {{ image_name }} /build 18 | register: build_cmd 19 | changed_when: not prev_hash.stdout or (prev_hash.stdout and prev_hash.stdout not in ''.join(build_cmd.stdout_lines[-2:])) 20 | 21 | - name: Converge 22 | hosts: localhost 23 | connection: local 24 | 25 | vars: 26 | ansible_python_interpreter: '{{ ansible_playbook_python }}' 27 | deploy_dir: "{{ lookup('env', 'MOLECULE_PROJECT_DIRECTORY') }}/deploy" 28 | pull_policy: Never 29 | operator_image: mcrouter.example.com/mcrouter-operator:testing 30 | custom_resource: "{{ lookup('file', '/'.join([deploy_dir, 'crds/mcrouter_v1alpha3_mcrouter_cr.yaml'])) | from_yaml }}" 31 | 32 | tasks: 33 | 34 | - block: 35 | 36 | - name: Delete the Operator Deployment 37 | k8s: 38 | state: absent 39 | namespace: '{{ namespace }}' 40 | definition: "{{ lookup('template', '/'.join([deploy_dir, 'operator.yaml'])) }}" 41 | register: delete_deployment 42 | when: hostvars[groups.k8s.0].build_cmd.changed 43 | 44 | - name: Wait 30s for Operator Deployment to terminate 45 | k8s_facts: 46 | api_version: '{{ definition.apiVersion }}' 47 | kind: '{{ definition.kind }}' 48 | namespace: '{{ namespace }}' 49 | name: '{{ definition.metadata.name }}' 50 | vars: 51 | definition: "{{ lookup('template', '/'.join([deploy_dir, 'operator.yaml'])) | from_yaml }}" 52 | register: deployment 53 | until: not deployment.resources 54 | delay: 3 55 | retries: 10 56 | when: delete_deployment.changed 57 | 58 | - name: Create the Operator Deployment 59 | k8s: 60 | namespace: '{{ namespace }}' 61 | definition: "{{ lookup('template', '/'.join([deploy_dir, 'operator.yaml'])) }}" 62 | 63 | - name: Create the mcrouter.example.com/v1alpha3.Mcrouter 64 | k8s: 65 | state: present 66 | namespace: '{{ namespace }}' 67 | definition: '{{ custom_resource }}' 68 | 69 | - name: Wait 5m for reconciliation to run 70 | k8s_facts: 71 | api_version: '{{ custom_resource.apiVersion }}' 72 | kind: '{{ custom_resource.kind }}' 73 | namespace: '{{ namespace }}' 74 | name: '{{ custom_resource.metadata.name }}' 75 | register: cr 76 | until: 77 | - "'Successful' in (cr | json_query('resources[].status.conditions[].reason'))" 78 | delay: 5 79 | retries: 60 80 | 81 | rescue: 82 | 83 | - name: debug cr 84 | ignore_errors: yes 85 | failed_when: false 86 | debug: 87 | var: debug_cr 88 | vars: 89 | debug_cr: '{{ lookup("k8s", 90 | kind=custom_resource.kind, 91 | api_version=custom_resource.apiVersion, 92 | namespace=namespace, 93 | resource_name=custom_resource.metadata.name 94 | )}}' 95 | 96 | - name: debug mcrouter deployment 97 | ignore_errors: yes 98 | failed_when: false 99 | debug: 100 | var: deploy 101 | vars: 102 | deploy: '{{ lookup("k8s", 103 | kind="Deployment", 104 | api_version="apps/v1", 105 | namespace=namespace, 106 | label_selector="app=mcrouter" 107 | )}}' 108 | 109 | - name: get operator logs 110 | ignore_errors: yes 111 | failed_when: false 112 | command: kubectl logs deployment/{{ definition.metadata.name }} -n {{ namespace }} 113 | environment: 114 | KUBECONFIG: '{{ lookup("env", "KUBECONFIG") }}' 115 | vars: 116 | definition: "{{ lookup('template', '/'.join([deploy_dir, 'operator.yaml'])) | from_yaml }}" 117 | register: log 118 | 119 | - name: print debug output 120 | debug: var=log.stdout_lines 121 | 122 | - name: fail if converge didn't succeed 123 | fail: 124 | msg: "Failed on action: converge" 125 | 126 | - import_playbook: '{{ playbook_dir }}/../default/asserts.yml' 127 | -------------------------------------------------------------------------------- /molecule/test-local/molecule.yml: -------------------------------------------------------------------------------- 1 | --- 2 | dependency: 3 | name: galaxy 4 | driver: 5 | name: docker 6 | lint: | 7 | set -e 8 | yamllint . 9 | ansible-lint 10 | platforms: 11 | - name: kind-test-local 12 | groups: 13 | - k8s 14 | image: bsycorp/kind:latest-1.15 15 | privileged: True 16 | override_command: no 17 | exposed_ports: 18 | - 8443/tcp 19 | - 10080/tcp 20 | published_ports: 21 | - 0.0.0.0:${TEST_CLUSTER_PORT:-10443}:8443/tcp 22 | pre_build_image: yes 23 | volumes: 24 | - ${MOLECULE_PROJECT_DIRECTORY}:/build:Z 25 | provisioner: 26 | name: ansible 27 | log: True 28 | inventory: 29 | group_vars: 30 | all: 31 | namespace: ${TEST_NAMESPACE:-default} 32 | env: 33 | K8S_AUTH_KUBECONFIG: /tmp/molecule/kind-test-local/kubeconfig 34 | KUBECONFIG: /tmp/molecule/kind-test-local/kubeconfig 35 | ANSIBLE_ROLES_PATH: ${MOLECULE_PROJECT_DIRECTORY}/roles 36 | KIND_PORT: '${TEST_CLUSTER_PORT:-10443}' 37 | scenario: 38 | test_sequence: 39 | - lint 40 | - destroy 41 | - dependency 42 | - syntax 43 | - create 44 | - prepare 45 | - converge 46 | - side_effect 47 | - verify 48 | - destroy 49 | -------------------------------------------------------------------------------- /molecule/test-local/prepare.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - import_playbook: ../default/prepare.yml 3 | 4 | - name: Prepare operator resources 5 | hosts: localhost 6 | connection: local 7 | 8 | vars: 9 | ansible_python_interpreter: '{{ ansible_playbook_python }}' 10 | deploy_dir: "{{ lookup('env', 'MOLECULE_PROJECT_DIRECTORY') }}/deploy" 11 | 12 | tasks: 13 | - name: Create Custom Resource Definition 14 | k8s: 15 | definition: "{{ lookup('file', '/'.join([deploy_dir, 'crds/mcrouter_v1alpha3_mcrouter_crd.yaml'])) }}" 16 | 17 | - name: Ensure specified namespace is present 18 | k8s: 19 | api_version: v1 20 | kind: Namespace 21 | name: '{{ namespace }}' 22 | 23 | - name: Create RBAC resources 24 | k8s: 25 | definition: "{{ lookup('template', '/'.join([deploy_dir, item])) }}" 26 | namespace: '{{ namespace }}' 27 | with_items: 28 | - role.yaml 29 | - role_binding.yaml 30 | - service_account.yaml 31 | -------------------------------------------------------------------------------- /requirements.yml: -------------------------------------------------------------------------------- 1 | --- 2 | collections: 3 | - community.kubernetes 4 | - operator_sdk.util 5 | -------------------------------------------------------------------------------- /roles/mcrouter/README.md: -------------------------------------------------------------------------------- 1 | Mcrouter role for Mcrouter Operator 2 | ========= 3 | 4 | An Ansible role which deploys Mcrouter and Memcached into a Kubernetes cluster. Meant to be used as part of the Mcrouter Operator. 5 | 6 | Requirements 7 | ------------ 8 | 9 | A Kubernetes cluster. 10 | 11 | Role Variables 12 | -------------- 13 | 14 | mcrouter_image: devan2502/mcrouter:latest 15 | 16 | The container image to be used to run Mcrouter 17 | 18 | mcrouter_port: 5000 19 | 20 | The port on which Mcrouter should be running. Used also to set the `containerPort` in the Mcrouter Deployment. 21 | 22 | memcached_image: memcached:1.5-alpine 23 | 24 | The container image to be used to run Memcached. 25 | 26 | memcached_pool_size: 2 27 | 28 | How many instances should be in the Memcached pool. 29 | 30 | memcached_port: 11211 31 | 32 | The port on which Memcached should be running. Used also to set the `containerPort` in the Memcached StatefulSet, and for the connection between Mcrouter and Memcached. 33 | 34 | pool_setup: replicated 35 | 36 | Can be one of `replicated` or `sharded`. 37 | 38 | - `replicated`: sends all Memcached `set`s to all Memcached pods, and distributes `get`s randomly. 39 | - `sharded`: uses a key hashing algorithm to distribute Memcached `set`s and `get`s among Memcached Pods; this means a key `foo` may always go to pod A, while the key `bar` always goes to pod B. 40 | 41 | debug_fifo_root: '' 42 | 43 | You can debug and inspect Mcrouter fifos using `mcpiper`, by setting `debug_fifo_root` to `/var/mcrouter/fifos`. Use `mcpiper` inside the mcrouter pod once it's reconfigured: `/usr/local/mcrouter/install/bin/mcpiper`. Note that you will not see any output (besides maybe an error message) until requests are sent to mcrouter. 44 | 45 | Dependencies 46 | ------------ 47 | 48 | None. 49 | 50 | Example Playbook 51 | ---------------- 52 | 53 | This role is meant to be used in an Ansible Operator inside a Kubernetes cluster, so it may not work as expected in typical Ansible playbooks. 54 | 55 | License 56 | ------- 57 | 58 | BSD 59 | 60 | Author Information 61 | ------------------ 62 | 63 | Anshul Behl (anshulbehl.19@gmail.com) 64 | John Lieske (@johnlieske) 65 | Jeff Geerling (@geerlingguy) 66 | -------------------------------------------------------------------------------- /roles/mcrouter/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | mcrouter_image: devan2502/mcrouter:latest 3 | mcrouter_port: 5000 4 | memcached_image: memcached:1.5-alpine 5 | memcached_pool_size: 2 6 | memcached_port: 11211 7 | pool_setup: replicated 8 | debug_fifo_root: '' 9 | -------------------------------------------------------------------------------- /roles/mcrouter/meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | galaxy_info: 3 | author: Anshul Behl, John Lieske, Jeff Geerling 4 | description: An Ansible role which deploys Mcrouter to Kubernetes 5 | company: Red Hat, Inc 6 | 7 | license: BSD 8 | 9 | min_ansible_version: 2.8 10 | 11 | platforms: 12 | - name: EL 13 | versions: 14 | - all 15 | - name: Debian 16 | versions: 17 | - all 18 | 19 | galaxy_tags: 20 | - kubernetes 21 | - operator 22 | - mcrouter 23 | - memcached 24 | 25 | dependencies: [] 26 | -------------------------------------------------------------------------------- /roles/mcrouter/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Create StatefulSet for Memcache 3 | k8s: 4 | definition: "{{ lookup('template', 'memcache_statefulset.yaml.j2')|from_yaml }}" 5 | 6 | - name: Create Service for Memcache 7 | k8s: 8 | definition: "{{ lookup('template', 'memcache_service.yaml.j2')|from_yaml }}" 9 | 10 | - name: Create Deployment for Mcrouter attaching memcache servers to its pool 11 | k8s: 12 | definition: "{{ lookup('template', 'mcrouter_deployment.yaml.j2')|from_yaml }}" 13 | -------------------------------------------------------------------------------- /roles/mcrouter/templates/mcrouter_deployment.yaml.j2: -------------------------------------------------------------------------------- 1 | kind: Deployment 2 | apiVersion: apps/v1 3 | metadata: 4 | name: '{{ meta.name }}' 5 | namespace: '{{ meta.namespace }}' 6 | spec: 7 | replicas: 1 8 | selector: 9 | matchLabels: 10 | app: mcrouter 11 | template: 12 | metadata: 13 | labels: 14 | app: mcrouter 15 | spec: 16 | containers: 17 | - name: mcrouter 18 | command: 19 | - mcrouter 20 | {% if pool_setup == 'sharded' %} 21 | - --config-str={"pools":{"A":{"servers":[{{ memcached_pool_str }}]}},"route":"PoolRoute|A"} 22 | {% elif pool_setup == 'replicated' %} 23 | - --config-str={"pools":{"A":{"servers":[{{ memcached_pool_str }}]}},"route":{"type":"OperationSelectorRoute","operation_policies":{"add":"AllSyncRoute|Pool|A","delete":"AllSyncRoute|Pool|A","get":"RandomRoute|Pool|A","set":"AllSyncRoute|Pool|A"}}} 24 | {% endif %} 25 | - -p 26 | - "{{ mcrouter_port }}" 27 | {% if debug_fifo_root != '' %} 28 | - --debug-fifo-root={{ debug_fifo_root }} 29 | {% endif %} 30 | image: '{{ mcrouter_image }}' 31 | ports: 32 | - containerPort: {{ mcrouter_port }} 33 | -------------------------------------------------------------------------------- /roles/mcrouter/templates/memcache_service.yaml.j2: -------------------------------------------------------------------------------- 1 | kind: Service 2 | apiVersion: v1 3 | metadata: 4 | name: '{{ meta.name }}-memcached' 5 | namespace: '{{ meta.namespace }}' 6 | spec: 7 | ports: 8 | - port: {{ memcached_port }} 9 | protocol: TCP 10 | selector: 11 | matchLabels: 12 | app: '{{ meta.name }}-cache' 13 | -------------------------------------------------------------------------------- /roles/mcrouter/templates/memcache_statefulset.yaml.j2: -------------------------------------------------------------------------------- 1 | kind: StatefulSet 2 | apiVersion: apps/v1 3 | metadata: 4 | name: '{{ meta.name }}-memcached' 5 | namespace: '{{ meta.namespace }}' 6 | spec: 7 | replicas: {{ memcached_pool_size|int }} 8 | serviceName: '{{ meta.name }}-memcached' 9 | selector: 10 | matchLabels: 11 | app: '{{ meta.name }}-cache' 12 | template: 13 | metadata: 14 | labels: 15 | app: '{{ meta.name }}-cache' 16 | spec: 17 | containers: 18 | - name: memcached 19 | command: ["memcached", "-m", "256", "-I", "32m", "-v"] 20 | image: '{{ memcached_image }}' 21 | ports: 22 | - containerPort: {{ memcached_port }} 23 | -------------------------------------------------------------------------------- /roles/mcrouter/vars/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | memcached_pool_str: '{% for item in range(memcached_pool_size|int) %}"{{ meta.name }}-memcached-{{ item }}.{{ meta.name }}-memcached.{{ meta.namespace }}.svc.cluster.local:{{ memcached_port }}"{% if not loop.last%},{% endif %}{% endfor %}' 3 | -------------------------------------------------------------------------------- /watches.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | - version: v1alpha3 3 | group: mcrouter.example.com 4 | kind: Mcrouter 5 | role: /opt/ansible/roles/mcrouter 6 | --------------------------------------------------------------------------------