├── teardown.sh ├── generate-secret.sh ├── LICENSE ├── README.md └── mongodb.yaml /teardown.sh: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /generate-secret.sh: -------------------------------------------------------------------------------- 1 | /usr/bin/openssl rand -base64 741 > mongosecret 2 | kubectl create secret generic mongosecret --from-file=mongosecret 3 | rm mongosecret -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Michael Green 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # kubernetes-mongodb 2 | Working mongodb replica sets with kubernetes and MiniKube. 3 | If you want this to work properly on GKE just change the storage class portion. 4 | 5 | 6 | You'll need a secret for your mongodb key if you don't have one already: `sh generate-secret.sh` 7 | 8 | ``` 9 | kubectl create -f mongodb.yaml 10 | ``` 11 | 12 | This will create a mongodb replica set across three nodes with an admin account that has credentials you define with environment variables. 13 | 14 | 15 | Other useful things: 16 | If you are having trouble deleting statefulsets while setting things up / testing (don't do this in production) 17 | ``` 18 | # Note you wouldn't want to do this normally. 19 | kubectl delete statefulsets mongod --force --grace-period=0 --cascade=false; 20 | kubectl delete services mongodb-service; 21 | kubectl delete pods all --grace-period=0 --force; 22 | ``` 23 | 24 | # Watch out, your stateful sets aren't guaranteed to be assigned the same node on restart 25 | This can lead to an issue when restarting clusters for an update (i.e: new kube version). I will create an automated fix for this when I have more time but for now you can just kill your other pods so they get assigned to the right things. what lol 26 | 27 | 28 | 29 | The majority of the yaml is based on: 30 | https://pauldone.blogspot.com/2017/06/deploying-mongodb-on-kubernetes-gke25.html 31 | 32 | The automatic configuration / pod introspection is written by me, which uses a different approach to configuring pods then scripts you have to call manually. 33 | 34 | This fixes some of the bugs that were present in that yaml were a pain to figure out. Figured I would share. This also works locally with minikube (version 0.26) independent of google cloud. 35 | -------------------------------------------------------------------------------- /mongodb.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: mongodb-service 5 | labels: 6 | name: mongo 7 | spec: 8 | ports: 9 | - port: 27017 10 | targetPort: 27017 11 | clusterIP: None 12 | selector: 13 | role: mongo 14 | --- 15 | apiVersion: apps/v1beta1 16 | kind: StatefulSet 17 | metadata: 18 | name: mongod 19 | spec: 20 | serviceName: mongodb-service 21 | replicas: 3 22 | template: 23 | metadata: 24 | labels: 25 | role: mongo 26 | environment: test 27 | replicaset: MainRepSet 28 | spec: 29 | affinity: 30 | # This is to prevent mongodb replicas from ending up on the same 31 | # host machine 32 | podAntiAffinity: 33 | preferredDuringSchedulingIgnoredDuringExecution: 34 | - weight: 100 35 | podAffinityTerm: 36 | labelSelector: 37 | matchExpressions: 38 | - key: replicaset 39 | operator: In 40 | values: 41 | - MainRepSet 42 | topologyKey: kubernetes.io/hostname 43 | terminationGracePeriodSeconds: 10 44 | containers: 45 | - name: mongod-container 46 | #image: pkdone/mongo-ent:3.4 47 | image: mongo 48 | imagePullPolicy: Always 49 | env: 50 | - name: STATEFULSET_NAME 51 | valueFrom: 52 | fieldRef: 53 | fieldPath: metadata.name 54 | - name: MACHINE_MEMORY 55 | value: &machineMemory "500Mi" 56 | # We can change this to a secret 57 | - name: MONGO_USER 58 | value: "admin" 59 | - name: MONGO_PASSWORD 60 | value: "abc123changeme" 61 | command: 62 | - "bash" 63 | - "-c" 64 | #the default cache size guidance is: "50% of RAM minus 1 GB, or 256 MB" 65 | # Which is why the wired tiger cache size is this way. This assumes 2gb of memory 66 | # on the host machine. The plan is to change 0.25 to a calculated value 67 | # by passing an environment variable that contains the host machines memory 68 | # and automatically "personalizing" mongo to the machine its running on 69 | - | 70 | numactl --interleave=all mongod --wiredTigerCacheSizeGB 0.25 --bind_ip 0.0.0.0 \ 71 | --replSet MainRepSet --auth --clusterAuthMode keyFile \ 72 | --keyFile /etc/secrets-volume/mongosecret \ 73 | --setParameter authenticationMechanisms=SCRAM-SHA-1 \ 74 | --fork --logpath /var/log/mongod.log \ 75 | && 76 | if [ $STATEFULSET_NAME == "mongod-0" ]; 77 | then 78 | initiated=0 79 | replica_exists=false 80 | while [ "$replica_exists" != true ] 81 | do 82 | # Need to check to make sure that the other pods are currently up 83 | initiated=`mongo --quiet --eval 'rs.initiate({_id: "MainRepSet", version: 1, members: [ 84 | { _id: 0, host : "mongod-0.mongodb-service.default.svc.cluster.local:27017" }, 85 | { _id: 1, host : "mongod-1.mongodb-service.default.svc.cluster.local:27017" }, 86 | { _id: 2, host : "mongod-2.mongodb-service.default.svc.cluster.local:27017" } 87 | ]})["ok"]'`; 88 | replica_exists=`mongo --quiet --eval 'db.isMaster()["ismaster"]'`; 89 | # Shit we need to check rs status i think instead 90 | echo "Replica exists: " $replica_exists; 91 | sleep 2 92 | done 93 | # If everything else is workign then we need to create an admin user 94 | mongo --eval "db.getSiblingDB('admin').createUser({ 95 | user : \"$MONGO_USER\", 96 | pwd : \"$MONGO_PASSWORD\", 97 | roles: [ { role: 'root', db: 'admin' } ] 98 | });" 99 | echo "STATEFULSET_NAME: $STATEFULSET_NAME \n MACHINE_MEMORY: $MACHINE_MEMORY"; 100 | fi && tailf /var/log/mongod.log 101 | # resources: 102 | # requests: 103 | # cpu: 1 104 | # memory: 300Mi 105 | ports: 106 | - containerPort: 27017 107 | volumeMounts: 108 | - name: secrets-volume 109 | # readOnly: true 110 | mountPath: /etc/secrets-volume/mongosecret 111 | subPath: mongosecret 112 | - name: mongodb-persistent-storage-claim 113 | mountPath: /data/db 114 | volumes: 115 | - name: secrets-volume 116 | secret: 117 | secretName: mongosecret 118 | # https://coderstoolbox.net/number/ 119 | # Convert to octal to decimal because kube only accepts 120 | # decimal as input 121 | defaultMode: 256 122 | volumeClaimTemplates: 123 | - metadata: 124 | name: mongodb-persistent-storage-claim 125 | annotations: 126 | volume.beta.kubernetes.io/storage-class: "fast" 127 | spec: 128 | accessModes: [ "ReadWriteOnce" ] 129 | resources: 130 | requests: 131 | storage: 500Mi 132 | # --- 133 | # Google cloud 134 | # kind: StorageClass 135 | # apiVersion: storage.k8s.io/v1beta1 136 | # metadata: 137 | # name: fast 138 | # provisioner: kubernetes.io/gce-pd 139 | # parameters: 140 | # type: pd-ssd 141 | # apiVersion: "v1" 142 | # --- 143 | # kind: "PersistentVolume" 144 | # metadata: 145 | # name: data-volume-INST 146 | # spec: 147 | # capacity: 148 | # storage: 10Gi 149 | # accessModes: 150 | # - ReadWriteOnce 151 | # persistentVolumeReclaimPolicy: Retain 152 | # storageClassName: fast 153 | # gcePersistentDisk: 154 | # fsType: xfs 155 | # pdName: pd-ssd-disk-INST 156 | --- 157 | # Local storage 158 | kind: StorageClass 159 | apiVersion: storage.k8s.io/v1beta1 160 | metadata: 161 | name: fast 162 | provisioner: k8s.io/minikube-hostpath --------------------------------------------------------------------------------