├── unit1
├── operator
│ ├── params.yaml
│ ├── templates
│ │ ├── config.yaml
│ │ └── pod.yaml
│ └── operator.yaml
└── README.md
├── unit2
├── operator
│ ├── params.yaml
│ ├── templates
│ │ ├── service.yaml
│ │ └── node.yaml
│ └── operator.yaml
└── README.md
├── unit3
├── operator
│ ├── params.yaml
│ ├── templates
│ │ ├── service.yaml
│ │ └── node.yaml
│ └── operator.yaml
├── operator-next
│ ├── params.yaml
│ ├── templates
│ │ ├── service.yaml
│ │ └── node.yaml
│ └── operator.yaml
└── README.md
├── unit4
├── operator
│ ├── params.yaml
│ ├── templates
│ │ ├── service.yaml
│ │ ├── restore.yaml
│ │ ├── backup.yaml
│ │ └── node.yaml
│ └── operator.yaml
├── backup-restore
│ ├── backup.yaml
│ └── restore.yaml
└── README.md
├── README.md
└── LICENSE
/unit1/operator/params.yaml:
--------------------------------------------------------------------------------
1 | WHO:
2 | default: "Ravi"
3 |
--------------------------------------------------------------------------------
/unit2/operator/params.yaml:
--------------------------------------------------------------------------------
1 | NODE_COUNT:
2 | default: "3"
3 |
--------------------------------------------------------------------------------
/unit3/operator/params.yaml:
--------------------------------------------------------------------------------
1 | NODE_COUNT:
2 | default: "3"
3 | TEST:
4 | default: "0"
5 | trigger: super
6 |
--------------------------------------------------------------------------------
/unit3/operator-next/params.yaml:
--------------------------------------------------------------------------------
1 | NODE_COUNT:
2 | default: "3"
3 | TEST:
4 | default: "0"
5 | trigger: super
6 |
--------------------------------------------------------------------------------
/unit1/operator/templates/config.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: v1
2 | kind: ConfigMap
3 | metadata:
4 | name: config
5 | namespace: {{ .Namespace }}
6 | data:
7 | index.html: |
8 | {{ .Params.WHO }}, hello from {{ .Name }} !!!
9 |
--------------------------------------------------------------------------------
/unit4/operator/params.yaml:
--------------------------------------------------------------------------------
1 | NODE_COUNT:
2 | default: "3"
3 | S3_ACCESS_KEY:
4 | default: "minio"
5 | S3_SECRET_KEY:
6 | default: "minio123"
7 | S3_ENDPOINT:
8 | default: "minio:9000"
9 | RESTORE_SNAPSHOT_ID:
10 | default: ""
11 |
--------------------------------------------------------------------------------
/unit2/operator/templates/service.yaml:
--------------------------------------------------------------------------------
1 | kind: Service
2 | apiVersion: v1
3 | metadata:
4 | name: hs
5 | namespace: {{ .Namespace }}
6 | spec:
7 | selector:
8 | app: elastic
9 | ports:
10 | - protocol: TCP
11 | port: 9200
12 | clusterIP: None
13 |
--------------------------------------------------------------------------------
/unit3/operator/templates/service.yaml:
--------------------------------------------------------------------------------
1 | kind: Service
2 | apiVersion: v1
3 | metadata:
4 | name: hs
5 | namespace: {{ .Namespace }}
6 | spec:
7 | selector:
8 | app: elastic
9 | ports:
10 | - protocol: TCP
11 | port: 9200
12 | clusterIP: None
13 |
--------------------------------------------------------------------------------
/unit4/operator/templates/service.yaml:
--------------------------------------------------------------------------------
1 | kind: Service
2 | apiVersion: v1
3 | metadata:
4 | name: hs
5 | namespace: {{ .Namespace }}
6 | spec:
7 | selector:
8 | app: elastic
9 | ports:
10 | - protocol: TCP
11 | port: 9200
12 | clusterIP: None
13 |
--------------------------------------------------------------------------------
/unit3/operator-next/templates/service.yaml:
--------------------------------------------------------------------------------
1 | kind: Service
2 | apiVersion: v1
3 | metadata:
4 | name: hs
5 | namespace: {{ .Namespace }}
6 | spec:
7 | selector:
8 | app: elastic
9 | ports:
10 | - protocol: TCP
11 | port: 9200
12 | clusterIP: None
13 |
--------------------------------------------------------------------------------
/unit1/operator/templates/pod.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: v1
2 | kind: Pod
3 | metadata:
4 | name: pod
5 | namespace: {{ .Namespace }}
6 | spec:
7 | containers:
8 | - name: nginx
9 | image: nginx
10 | ports:
11 | - containerPort: 80
12 | name: web
13 | volumeMounts:
14 | - name: config
15 | mountPath: /usr/share/nginx/html
16 | volumes:
17 | - name: config
18 | configMap:
19 | name: {{ .Name }}-config
20 |
--------------------------------------------------------------------------------
/unit4/backup-restore/backup.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: kudo.dev/v1alpha1
2 | kind: PlanExecution
3 | metadata:
4 | name: backup
5 | namespace: default
6 | ownerReferences:
7 | - apiVersion: kudo.dev/v1alpha1
8 | blockOwnerDeletion: true
9 | controller: true
10 | kind: Instance
11 | name: myes
12 | uid: e4c10faa-727d-11e9-80a1-0a97366f7696
13 | spec:
14 | instance:
15 | kind: Instance
16 | name: myes
17 | namespace: default
18 | planName: backup
19 |
--------------------------------------------------------------------------------
/unit4/backup-restore/restore.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: kudo.dev/v1alpha1
2 | kind: PlanExecution
3 | metadata:
4 | name: restore
5 | namespace: default
6 | ownerReferences:
7 | - apiVersion: kudo.dev/v1alpha1
8 | blockOwnerDeletion: true
9 | controller: true
10 | kind: Instance
11 | name: myes
12 | uid: e4c10faa-727d-11e9-80a1-0a97366f7696
13 | spec:
14 | instance:
15 | kind: Instance
16 | name: myes
17 | namespace: default
18 | planName: restore
19 |
--------------------------------------------------------------------------------
/unit4/operator/templates/restore.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: batch/v1
2 | kind: Job
3 | metadata:
4 | name: {{ .PlanName }}-job
5 | namespace: default
6 | spec:
7 | template:
8 | metadata:
9 | name: {{ .PlanName }}-job
10 | spec:
11 | restartPolicy: OnFailure
12 | containers:
13 | - name: restore
14 | image: centos:7
15 | command:
16 | - sh
17 | - -c
18 | - |
19 | curl -X POST "myes-node-0.myes-hs:9200/_snapshot/my_s3_repository/{{ .Params.RESTORE_SNAPSHOT_ID }}/_restore?pretty"
20 |
--------------------------------------------------------------------------------
/unit1/operator/operator.yaml:
--------------------------------------------------------------------------------
1 | name: "myservice"
2 | version: "0.1.0"
3 | kudoVersion: 0.7.2
4 | kubernetesVersion: 1.15.0
5 | maintainers:
6 | - name: Michael Beisiegel
7 | email: michael.beisiegel@gmail.com
8 | url: https://github.com/realmbgl/kudo-tutorial
9 | tasks:
10 | deploy-task:
11 | resources:
12 | - config.yaml
13 | - pod.yaml
14 | plans:
15 | deploy:
16 | strategy: serial
17 | phases:
18 | - name: deploy-phase
19 | strategy: parallel
20 | steps:
21 | - name: deploy-step
22 | tasks:
23 | - deploy-task
24 |
--------------------------------------------------------------------------------
/unit2/operator/operator.yaml:
--------------------------------------------------------------------------------
1 | name: "elastic"
2 | version: "0.1.0"
3 | kudoVersion: 0.7.2
4 | kubernetesVersion: 1.15.0
5 | appVersion: 7.0.0
6 | maintainers:
7 | - name: Michael Beisiegel
8 | email: michael.beisiegel@gmail.com
9 | url: https://github.com/realmbgl/kudo-tutorial
10 | tasks:
11 | deploy-task:
12 | resources:
13 | - service.yaml
14 | - node.yaml
15 | plans:
16 | deploy:
17 | strategy: serial
18 | phases:
19 | - name: deploy-phase
20 | strategy: parallel
21 | steps:
22 | - name: deploy-step
23 | tasks:
24 | - deploy-task
25 |
--------------------------------------------------------------------------------
/unit4/operator/templates/backup.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: batch/v1
2 | kind: Job
3 | metadata:
4 | name: {{ .PlanName }}-job
5 | namespace: default
6 | spec:
7 | template:
8 | metadata:
9 | name: {{ .PlanName }}-job
10 | spec:
11 | restartPolicy: OnFailure
12 | containers:
13 | - name: backup
14 | image: centos:7
15 | command:
16 | - sh
17 | - -c
18 | - |
19 | curl -X PUT "myes-node-0.myes-hs:9200/_snapshot/my_s3_repository" -H 'Content-Type: application/json' -d'
20 | {
21 | "type": "s3",
22 | "settings": {
23 | "bucket": "es-bucket",
24 | "endpoint": "{{ .Params.S3_ENDPOINT }}",
25 | "protocol": "http"
26 | }
27 | }
28 | ';
29 | TS=$(date +%s);
30 | curl -X PUT "myes-node-0.myes-hs:9200/_snapshot/my_s3_repository/snapshot_$TS?wait_for_completion=true&pretty"
31 |
--------------------------------------------------------------------------------
/unit4/operator/operator.yaml:
--------------------------------------------------------------------------------
1 | name: "elastic"
2 | version: "0.1.0"
3 | kudoVersion: 0.7.2
4 | kubernetesVersion: 1.15.0
5 | appVersion: 7.0.0
6 | maintainers:
7 | - name: Michael Beisiegel
8 | email: michael.beisiegel@gmail.com
9 | url: https://github.com/realmbgl/kudo-tutorial
10 | tasks:
11 | deploy-task:
12 | resources:
13 | - service.yaml
14 | - node.yaml
15 | backup-task:
16 | resources:
17 | - backup.yaml
18 | restore-task:
19 | resources:
20 | - restore.yaml
21 | plans:
22 | deploy:
23 | strategy: serial
24 | phases:
25 | - name: deploy-phase
26 | strategy: parallel
27 | steps:
28 | - name: deploy-step
29 | tasks:
30 | - deploy-task
31 | backup:
32 | strategy: serial
33 | phases:
34 | - name: backup-phase
35 | strategy: serial
36 | steps:
37 | - name: backup-step
38 | tasks:
39 | - backup-task
40 | restore:
41 | strategy: serial
42 | phases:
43 | - name: restore-phase
44 | strategy: serial
45 | steps:
46 | - name: restore-step
47 | tasks:
48 | - restore-task
49 |
--------------------------------------------------------------------------------
/unit3/operator/operator.yaml:
--------------------------------------------------------------------------------
1 | name: "elastic"
2 | version: "0.1.0"
3 | kudoVersion: 0.7.2
4 | kubernetesVersion: 1.15.0
5 | appVersion: 7.0.0
6 | maintainers:
7 | - name: Michael Beisiegel
8 | email: michael.beisiegel@gmail.com
9 | url: https://github.com/realmbgl/kudo-tutorial
10 | tasks:
11 | deploy-task:
12 | resources:
13 | - service.yaml
14 | - node.yaml
15 | update-task:
16 | resources:
17 | - service.yaml
18 | - node.yaml
19 | upgrade-task:
20 | resources:
21 | - service.yaml
22 | - node.yaml
23 | super-task:
24 | resources:
25 | - service.yaml
26 | - node.yaml
27 | plans:
28 | deploy:
29 | strategy: serial
30 | phases:
31 | - name: deploy-phase
32 | strategy: parallel
33 | steps:
34 | - name: deploy-step
35 | tasks:
36 | - deploy-task
37 | update:
38 | strategy: serial
39 | phases:
40 | - name: update-phase
41 | strategy: parallel
42 | steps:
43 | - name: update-step
44 | tasks:
45 | - update-task
46 | upgrade:
47 | strategy: serial
48 | phases:
49 | - name: upgrade-phase
50 | strategy: parallel
51 | steps:
52 | - name: upgrade-step
53 | tasks:
54 | - upgrade-task
55 | super:
56 | strategy: serial
57 | phases:
58 | - name: super-phase
59 | strategy: parallel
60 | steps:
61 | - name: super-step
62 | tasks:
63 | - super-task
64 |
--------------------------------------------------------------------------------
/unit3/operator-next/operator.yaml:
--------------------------------------------------------------------------------
1 | name: "elastic"
2 | version: "0.2.0"
3 | kudoVersion: 0.7.2
4 | kubernetesVersion: 1.15.0
5 | appVersion: 7.2.0
6 | maintainers:
7 | - name: Michael Beisiegel
8 | email: michael.beisiegel@gmail.com
9 | url: https://github.com/realmbgl/kudo-tutorial
10 | tasks:
11 | deploy-task:
12 | resources:
13 | - service.yaml
14 | - node.yaml
15 | update-task:
16 | resources:
17 | - service.yaml
18 | - node.yaml
19 | upgrade-task:
20 | resources:
21 | - service.yaml
22 | - node.yaml
23 | super-task:
24 | resources:
25 | - service.yaml
26 | - node.yaml
27 | plans:
28 | deploy:
29 | strategy: serial
30 | phases:
31 | - name: deploy-phase
32 | strategy: parallel
33 | steps:
34 | - name: deploy-step
35 | tasks:
36 | - deploy-task
37 | update:
38 | strategy: serial
39 | phases:
40 | - name: update-phase
41 | strategy: parallel
42 | steps:
43 | - name: update-step
44 | tasks:
45 | - update-task
46 | upgrade:
47 | strategy: serial
48 | phases:
49 | - name: upgrade-phase
50 | strategy: parallel
51 | steps:
52 | - name: upgrade-step
53 | tasks:
54 | - upgrade-task
55 | super:
56 | strategy: serial
57 | phases:
58 | - name: super-phase
59 | strategy: parallel
60 | steps:
61 | - name: super-step
62 | tasks:
63 | - super-task
64 |
--------------------------------------------------------------------------------
/unit2/operator/templates/node.yaml:
--------------------------------------------------------------------------------
1 | kind: StatefulSet
2 | apiVersion: apps/v1
3 | metadata:
4 | name: node
5 | namespace: {{ .Namespace }}
6 | spec:
7 | selector:
8 | matchLabels:
9 | app: elastic # has to match .spec.template.metadata.labels
10 | serviceName: {{ .Name }}-hs
11 | replicas: {{ .Params.NODE_COUNT }}
12 | template:
13 | metadata:
14 | labels:
15 | app: elastic # has to match .spec.selector.matchLabels
16 | spec:
17 | initContainers:
18 | - name: init-sysctl
19 | image: busybox
20 | command: ['sh', '-c', 'sysctl -w vm.max_map_count=262144']
21 | securityContext:
22 | privileged: true
23 | - name: volume-permissions
24 | image: busybox
25 | command: ['sh', '-c', 'chown -R 1000:1000 /usr/share/elasticsearch/data']
26 | volumeMounts:
27 | - name: data
28 | mountPath: /usr/share/elasticsearch/data
29 | terminationGracePeriodSeconds: 10
30 | containers:
31 | - name: elastic
32 | image: elasticsearch:7.0.0
33 | ports:
34 | - containerPort: 9200
35 | name: api
36 | - containerPort: 9300
37 | name:
38 | env:
39 | - name: cluster.name
40 | value: {{ .Name }}-cluster
41 | - name: discovery.seed_hosts
42 | value: {{ .Name }}-node-0.{{ .Name }}-hs,{{ .Name }}-node-1.{{ .Name }}-hs,{{ .Name }}-node-2.{{ .Name }}-hs
43 | - name: cluster.initial_master_nodes
44 | value: {{ .Name }}-node-0,{{ .Name }}-node-1,{{ .Name }}-node-2
45 | volumeMounts:
46 | - name: data
47 | mountPath: /usr/share/elasticsearch/data
48 | volumeClaimTemplates:
49 | - metadata:
50 | name: data
51 | spec:
52 | accessModes: [ "ReadWriteOnce" ]
53 | resources:
54 | requests:
55 | storage: 1Gi
56 |
--------------------------------------------------------------------------------
/unit3/operator/templates/node.yaml:
--------------------------------------------------------------------------------
1 | kind: StatefulSet
2 | apiVersion: apps/v1
3 | metadata:
4 | name: node
5 | namespace: {{ .Namespace }}
6 | spec:
7 | selector:
8 | matchLabels:
9 | app: elastic # has to match .spec.template.metadata.labels
10 | serviceName: {{ .Name }}-hs
11 | replicas: {{ .Params.NODE_COUNT }}
12 | template:
13 | metadata:
14 | labels:
15 | app: elastic # has to match .spec.selector.matchLabels
16 | spec:
17 | initContainers:
18 | - name: init-sysctl
19 | image: busybox
20 | command: ['sh', '-c', 'sysctl -w vm.max_map_count=262144']
21 | securityContext:
22 | privileged: true
23 | - name: volume-permissions
24 | image: busybox
25 | command: ['sh', '-c', 'chown -R 1000:1000 /usr/share/elasticsearch/data']
26 | volumeMounts:
27 | - name: data
28 | mountPath: /usr/share/elasticsearch/data
29 | terminationGracePeriodSeconds: 10
30 | containers:
31 | - name: elastic
32 | image: elasticsearch:7.0.0
33 | ports:
34 | - containerPort: 9200
35 | name: api
36 | - containerPort: 9300
37 | name:
38 | env:
39 | - name: cluster.name
40 | value: {{ .Name }}-cluster
41 | - name: discovery.seed_hosts
42 | value: {{ .Name }}-node-0.{{ .Name }}-hs,{{ .Name }}-node-1.{{ .Name }}-hs,{{ .Name }}-node-2.{{ .Name }}-hs
43 | - name: cluster.initial_master_nodes
44 | value: {{ .Name }}-node-0,{{ .Name }}-node-1,{{ .Name }}-node-2
45 | volumeMounts:
46 | - name: data
47 | mountPath: /usr/share/elasticsearch/data
48 | volumeClaimTemplates:
49 | - metadata:
50 | name: data
51 | spec:
52 | accessModes: [ "ReadWriteOnce" ]
53 | resources:
54 | requests:
55 | storage: 1Gi
56 |
--------------------------------------------------------------------------------
/unit3/operator-next/templates/node.yaml:
--------------------------------------------------------------------------------
1 | kind: StatefulSet
2 | apiVersion: apps/v1
3 | metadata:
4 | name: node
5 | namespace: {{ .Namespace }}
6 | spec:
7 | selector:
8 | matchLabels:
9 | app: elastic # has to match .spec.template.metadata.labels
10 | serviceName: {{ .Name }}-hs
11 | replicas: {{ .Params.NODE_COUNT }}
12 | template:
13 | metadata:
14 | labels:
15 | app: elastic # has to match .spec.selector.matchLabels
16 | spec:
17 | initContainers:
18 | - name: init-sysctl
19 | image: busybox
20 | command: ['sh', '-c', 'sysctl -w vm.max_map_count=262144']
21 | securityContext:
22 | privileged: true
23 | - name: volume-permissions
24 | image: busybox
25 | command: ['sh', '-c', 'chown -R 1000:1000 /usr/share/elasticsearch/data']
26 | volumeMounts:
27 | - name: data
28 | mountPath: /usr/share/elasticsearch/data
29 | terminationGracePeriodSeconds: 10
30 | containers:
31 | - name: elastic
32 | image: elasticsearch:7.2.0
33 | ports:
34 | - containerPort: 9200
35 | name: api
36 | - containerPort: 9300
37 | name:
38 | env:
39 | - name: cluster.name
40 | value: {{ .Name }}-cluster
41 | - name: discovery.seed_hosts
42 | value: {{ .Name }}-node-0.{{ .Name }}-hs,{{ .Name }}-node-1.{{ .Name }}-hs,{{ .Name }}-node-2.{{ .Name }}-hs
43 | - name: cluster.initial_master_nodes
44 | value: {{ .Name }}-node-0,{{ .Name }}-node-1,{{ .Name }}-node-2
45 | volumeMounts:
46 | - name: data
47 | mountPath: /usr/share/elasticsearch/data
48 | volumeClaimTemplates:
49 | - metadata:
50 | name: data
51 | spec:
52 | accessModes: [ "ReadWriteOnce" ]
53 | resources:
54 | requests:
55 | storage: 1Gi
56 |
--------------------------------------------------------------------------------
/unit4/operator/templates/node.yaml:
--------------------------------------------------------------------------------
1 | kind: StatefulSet
2 | apiVersion: apps/v1
3 | metadata:
4 | name: node
5 | namespace: {{ .Namespace }}
6 | spec:
7 | selector:
8 | matchLabels:
9 | app: elastic # has to match .spec.template.metadata.labels
10 | serviceName: {{ .Name }}-hs
11 | replicas: {{ .Params.NODE_COUNT }}
12 | template:
13 | metadata:
14 | labels:
15 | app: elastic # has to match .spec.selector.matchLabels
16 | spec:
17 | initContainers:
18 | - name: init-sysctl
19 | image: busybox
20 | command: ['sh', '-c', 'sysctl -w vm.max_map_count=262144']
21 | securityContext:
22 | privileged: true
23 | - name: volume-permissions
24 | image: busybox
25 | command: ['sh', '-c', 'chown -R 1000:1000 /usr/share/elasticsearch/data']
26 | volumeMounts:
27 | - name: data
28 | mountPath: /usr/share/elasticsearch/data
29 | terminationGracePeriodSeconds: 10
30 | containers:
31 | - name: elastic
32 | image: elasticsearch:7.0.0
33 | command:
34 | - sh
35 | - -c
36 | - |
37 | /usr/share/elasticsearch/bin/elasticsearch-plugin install repository-s3 -b;
38 | /usr/share/elasticsearch/bin/elasticsearch-keystore create
39 | echo {{ .Params.S3_ACCESS_KEY }} | /usr/share/elasticsearch/bin/elasticsearch-keystore add --stdin s3.client.default.access_key;
40 | echo {{ .Params.S3_SECRET_KEY }} | /usr/share/elasticsearch/bin/elasticsearch-keystore add --stdin s3.client.default.secret_key;
41 | /usr/local/bin/docker-entrypoint.sh eswrapper
42 | ports:
43 | - containerPort: 9200
44 | name: api
45 | - containerPort: 9300
46 | name:
47 | env:
48 | - name: cluster.name
49 | value: {{ .Name }}-cluster
50 | - name: discovery.seed_hosts
51 | value: {{ .Name }}-node-0.{{ .Name }}-hs,{{ .Name }}-node-1.{{ .Name }}-hs,{{ .Name }}-node-2.{{ .Name }}-hs
52 | - name: cluster.initial_master_nodes
53 | value: {{ .Name }}-node-0,{{ .Name }}-node-1,{{ .Name }}-node-2
54 | volumeMounts:
55 | - name: data
56 | mountPath: /usr/share/elasticsearch/data
57 | volumeClaimTemplates:
58 | - metadata:
59 | name: data
60 | spec:
61 | accessModes: [ "ReadWriteOnce" ]
62 | resources:
63 | requests:
64 | storage: 1Gi
65 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # kudo tutorial
2 |
3 | 
4 |
5 | Developing `Kubernetes` operators using `KUDO`, the Kubernetes Universal Declarative Operator. KUDO provides a declarative approach to building production-grade Kubernetes Operators covering the entire application lifecycle.
6 |
7 | ## install kudo
8 |
9 | Kudo comes with its own `kubectl CLI plugin`. On `Mac OS X` you can install it using `brew`.
10 |
11 | ```
12 | brew tap kudobuilder/tap
13 | brew install kudo-cli
14 | ```
15 |
16 | More on the kudo CLI at [kudo.dev](https://kudo.dev/docs/cli/) .
17 |
18 | Once you have a running `kubernetes` cluster with `kubectl` installed along with the KUDO CLI plugin, you can install kudo as follows.
19 |
20 | ```
21 | kubectl kudo init
22 | ```
23 |
24 | More on the kudo install at [kudo.dev](https://kudo.dev/docs/getting-started/) .
25 |
26 |
27 | ## install and operate kudo operators
28 |
29 | Install and operate your first kudo operator using [kafka](https://github.com/kudobuilder/operators/blob/master/repository/kafka/docs/v0.2/install.md) as the sample.
30 |
31 |
32 | ## develop kudo operators
33 |
34 | ### kudo operator programming model
35 |
36 | As a developer of a kudo operator you have to author three types of `YAML` artifacts in the following file folder structure.
37 |
38 | ```
39 | operator.yaml
40 | params.yaml
41 | templates/
42 | .yaml
43 | ...
44 | ```
45 |
46 | In the following we describe the roles that each of the YAML artifacts plays and what they capture.
47 |
48 | #### operator.yaml
49 | The operator file defines the operator behavior.
50 |
51 | It has a `name` (e.g kafka), and a `version`.
52 |
53 | `Tasks` list the resource templates that get applied together.
54 |
55 | `Plans` orchestrate tasks through phases and steps. `Plans` consists of one or more `phases`. `Phases` consists of one or more `steps`. `Steps` contain one or more `tasks`. Both phases and also steps can be configured with an `execution strategy`, either `serial` or `parallel`.
56 |
57 | #### params.yaml
58 | The params file defines the parameters that can be used to configure an instance created by the operator. A parameter definition has a `name`, `default value`, `display name`, and `description`.
59 |
60 | #### template.yaml
61 | A template file defines the resources that can be applied by operator tasks. Samples are `config maps`, `service`, `deployment`, `stateful set`, ... .
62 |
63 |
64 | ### [unit 1](unit1): parameters, templates, tasks, and plans in action
65 | * showcasing the core operator concepts
66 |
67 | ### [unit 2](unit2): a stateful service
68 | * showcasing stateful set, and headless service templates
69 | * showcasing containers, and init containers
70 | * showcasing persistent volumes
71 | * showcasing instance update, showing scaling
72 |
73 | ### [unit 3](unit3): update and upgrade plans
74 | * showcasing update and upgrade Plans
75 | * showcasing controlled parameter updates
76 |
77 | ### [unit 4](unit4): custom plans
78 | * showcasing backup and restore
79 |
--------------------------------------------------------------------------------
/unit1/README.md:
--------------------------------------------------------------------------------
1 | ## unit 1: parameters, templates, tasks, and plans in action
2 |
3 | This unit uses an [operator sample](operator) to showcase the key kudo operator concepts parameters, templates, tasks, and plans.
4 |
5 | ### [params.yaml](operator/params.yaml)
6 |
7 | Parameters allow for the configuration of instances created by the operator
8 |
9 | The sample has one configuration parameter named `WHO` with a default value of `Ravi`.
10 |
11 | ```yaml
12 | WHO:
13 | default: "Ravi"
14 | ```
15 |
16 | Besides the keys shown in the sample a parameter can have the following additional keys `description`, `displayName`, `required`, and `trigger`.
17 |
18 | ### [templates](operator/templates)
19 |
20 | Templates define the resources that can be applied by this operator.
21 |
22 | The sample has two resource templates, one is of type `ConfigMap` the other of type `POD`. The pod runs an nginx container into which the config map which holds an index.html file is mounted.
23 |
24 | In the config map you see how parameters get templated in, here the parameter `{{ .Params.WHO }}`. There is also `{{ .NAMESPACE }}` and `{{ .NAME }}` which get provided by the operator on instantiation.
25 |
26 | [config.yaml](operator/config.yaml)
27 | ```yaml
28 | apiVersion: v1
29 | kind: ConfigMap
30 | metadata:
31 | name: config
32 | namespace: {{ .Namespace }}
33 | data:
34 | index.html: |
35 | {{ .Params.WHO }}, hello from {{ .Name }} !!!
36 | ```
37 |
38 | [pod.yaml](operator/pod.yaml)
39 | ```yaml
40 | apiVersion: v1
41 | kind: Pod
42 | metadata:
43 | name: pod
44 | namespace: {{ .Namespace }}
45 | spec:
46 | containers:
47 | - name: nginx
48 | image: nginx
49 | ports:
50 | - containerPort: 80
51 | name: web
52 | volumeMounts:
53 | - name: config
54 | mountPath: /usr/share/nginx/html
55 | volumes:
56 | - name: config
57 | configMap:
58 | name: {{ .Name }}-config
59 | ```
60 |
61 | ### [operator.yaml](operator/operator.yaml)
62 |
63 | #### general
64 |
65 | In the general section the `name` of the operator and its `version` is defined. It also lists the `kudo version` the operator works with.
66 |
67 | ```
68 | name: "myservice"
69 | version: "0.1.0"
70 | kudoVersion: 0.5.0
71 | ```
72 |
73 | #### tasks
74 |
75 | Tasks list the resource templates that get applied together.
76 |
77 | The sample `deploy-task` applies the `config` and `pod` template.
78 |
79 | ```yaml
80 | tasks:
81 | deploy-task:
82 | resources:
83 | - config.yaml
84 | - pod.yaml
85 | ```
86 |
87 | #### plans
88 |
89 | Plans orchestrate tasks through phases and steps.
90 |
91 | `Plans` consists of one or more `phases`. `Phases` consists of one or more `steps`. `Steps` contain one or more `tasks`. Both phases and also steps can be configured with an execution `strategy`, either serial or parallel.
92 |
93 | The sample has a `deploy` plan with a `deploy-phase` and a `deploy-step`. From the `deploy-step` the `deploy-task` is referenced. This task gets executed when an instance is created using the operator.
94 |
95 | ```yaml
96 | plans:
97 | deploy:
98 | strategy: serial
99 | phases:
100 | - name: deploy-phase
101 | strategy: parallel
102 | steps:
103 | - name: deploy-step
104 | tasks:
105 | - deploy-task
106 | ```
107 |
108 |
109 | ### Run the framework instance
110 |
111 | If you haven't already then clone the `kudo-tutorial` repository.
112 |
113 | ```
114 | git clone https://github.com/realmbgl/kudo-tutorial.git
115 | ```
116 |
117 | From the `unit1/operator` folder use the following command to install the operator and create an instance.
118 |
119 | ```
120 | kubectl kudo install . --instance myservice --parameter WHO=Matt
121 | ```
122 |
123 | Next enable localhost access to the instance.
124 |
125 | ```
126 | kubectl port-forward myservice-pod 8080:80
127 | ```
128 |
129 | [Click](http://localhost:8080/) to access the instance from your browser.
130 |
131 | You should see the following
132 |
133 | ```
134 | Matt, hello from myservice !!!
135 | ```
136 |
--------------------------------------------------------------------------------
/unit3/README.md:
--------------------------------------------------------------------------------
1 | ## unit 3: update and upgrade plans
2 |
3 | This unit builds on `unit 2` showcasing how `update`, and `upgrade` plans get triggered. It also showcases how `controlled parameter update` can be triggered.
4 |
5 |
6 | ### [params.yaml](operator/params.yaml)
7 |
8 | The sample has an additional `TEST` parameter, to showcase `controlled parameter update`. The `trigger` key of the `TEST` parameter points to the `super` plan to be executed on update.
9 |
10 | ```yaml
11 | NODE_COUNT:
12 | default: "3"
13 | TEST:
14 | default: "0"
15 | trigger: super
16 | ```
17 |
18 | ### [templates](operator/templates)
19 |
20 | No changes here.
21 |
22 | ### [operator.yaml](operator/operator.yaml)
23 |
24 | #### tasks
25 |
26 | Besides the `deploy-task` there is now also an `update-task`, `upgrade-task`, and `super-task`. At this point they all apply the same templates. This will change on further iteration, for example the `upgrade-task` will take care of the [elasticsearch rolling upgrade](https://www.elastic.co/guide/en/elasticsearch/reference/current/rolling-upgrades.html) steps.
27 |
28 | ```yaml
29 | deploy-task:
30 | resources:
31 | - service.yaml
32 | - node.yaml
33 | update-task:
34 | resources:
35 | - service.yaml
36 | - node.yaml
37 | upgrade-task:
38 | resources:
39 | - service.yaml
40 | - node.yaml
41 | super-task:
42 | resources:
43 | - service.yaml
44 | - node.yaml
45 | ```
46 |
47 | #### plans
48 |
49 | The sample has not only the `deploy` plan, but also `update`, `upgrade`, and a `super` plans.
50 |
51 | ```yaml
52 | plans:
53 | deploy:
54 | strategy: serial
55 | phases:
56 | - name: deploy-phase
57 | strategy: parallel
58 | steps:
59 | - name: deploy-step
60 | tasks:
61 | - deploy-task
62 | update:
63 | strategy: serial
64 | phases:
65 | - name: update-phase
66 | strategy: parallel
67 | steps:
68 | - name: update-step
69 | tasks:
70 | - update-task
71 | upgrade:
72 | strategy: serial
73 | phases:
74 | - name: upgrade-phase
75 | strategy: parallel
76 | steps:
77 | - name: upgrade-step
78 | tasks:
79 | - upgrade-task
80 | super:
81 | strategy: serial
82 | phases:
83 | - name: super-phase
84 | strategy: parallel
85 | steps:
86 | - name: super-step
87 | tasks:
88 | - super-task
89 | ```
90 |
91 |
92 | ### Run the framework instance
93 |
94 | If you haven't already then clone the `kudo-tutorial` repository.
95 |
96 | ```
97 | git clone https://github.com/realmbgl/kudo-tutorial.git
98 | ```
99 |
100 | From the `unit3/operator` folder use the following command to run the instance.
101 |
102 | ```
103 | kubectl kudo install . --instance myes
104 | ```
105 |
106 | Once the install is finished we should see the following pods.
107 | ```
108 | kubectl get pods
109 | NAME READY STATUS RESTARTS AGE
110 | myes-node-0 1/1 Running 0 42m
111 | myes-node-1 1/1 Running 0 41m
112 | myes-node-2 1/1 Running 0 41m
113 | ```
114 |
115 | Lets check on the plan execution history using the kudo cli, we see the `deploy plan` has been executed.
116 |
117 | ```
118 | kubectl kudo plan history --instance=myes
119 | History of all plan-executions for instance "myes" in namespace "default":
120 | .
121 | └── myes-deploy-206753060 (created 4h18m37s ago)
122 | ```
123 |
124 |
125 | ### Update the framework instance to scale to 4 nodes
126 |
127 | Update the `NODE_COUNT` parameter to `4`.
128 |
129 | From the `unit3/operator` folder use the following command to update the instance.
130 |
131 | ```
132 | kubectl kudo update --instance myes -p NODE_COUNT=4
133 | ```
134 |
135 | Once the update is finished we should see an additional pod `myes-node-3`.
136 |
137 | ```
138 | kubectl get pods
139 | NAME READY STATUS RESTARTS AGE
140 | myes-node-0 1/1 Running 0 46m
141 | myes-node-1 1/1 Running 0 46m
142 | myes-node-2 1/1 Running 0 45m
143 | myes-node-3 1/1 Running 0 70s
144 |
145 | ```
146 |
147 | Lets check on the plan execution history using the kudo cli, we see the `update plan` has been executed.
148 |
149 | ```
150 | kubectl kudo plan history --instance=myes
151 | History of all plan-executions for instance "myes" in namespace "default":
152 | .
153 | ├── myes-deploy-206753060 (created 4h20m36s ago)
154 | └── myes-update-207626186 (created 39s ago)
155 | ```
156 |
157 |
158 | ### Upgrade the framework instance to use a newer framework implementation
159 |
160 | Go to the `unit3/operator-next` folder. Its a copy of `unit3/operator` but with the operator version set to `0.2.0` in `operator.yaml` and using `elasticsearch:7.2.0` for the image in `node.yaml`.
161 |
162 | From the `unit3/operator-next` folder use the following command to upgrade the operator version.
163 |
164 | ```
165 | kubectl kudo upgrade . --instance myes
166 |
167 | operatorversion.kudo.dev/v1alpha1/elastic-0.2.0 successfully created
168 | instance./myes successfully updated
169 | ```
170 |
171 | Check on the versions available.
172 |
173 | ```
174 | kubectl get operatorversion
175 | NAME AGE
176 | elastic-0.1.0 33m
177 | elastic-0.2.0 4s
178 | ```
179 |
180 | Lets check on the plan execution history using the kudo cli, we see the `upgrade plan` has been executed.
181 |
182 | ```
183 | kubectl kudo plan history --instance=myes
184 | History of all plan-executions for instance "myes" in namespace "default":
185 | .
186 | ├── myes-deploy-206753060 (created 4h22m0s ago)
187 | ├── myes-update-207626186 (created 2m3s ago)
188 | └── myes-upgrade-521226614 (created 17s ago)
189 | ```
190 |
191 | Lets see whether the myes pods use the newer `elasticsearch container image`, you should see that version `7.2.0` is used after the upgrade.
192 |
193 | ```
194 | kubectl get pod myes-node-0 -o yaml | grep "docker.io/library/elasticsearch:"
195 | image: docker.io/library/elasticsearch:7.2.0
196 | ```
197 |
198 | ### Controlled parameter update
199 |
200 | Update the `TEST` parameter to like `100`.
201 |
202 | Use the following command to update the instance.
203 |
204 | ```
205 | kubectl kudo update myes -p TEST=100
206 | ```
207 |
208 | Lets check on the plan execution history using the kudo cli, we see the `super plan` has been executed.
209 |
210 | ```
211 | kubectl kudo plan history --instance=myes
212 | History of all plan-executions for instance "myes" in namespace "default":
213 | .
214 | ├── myes-deploy-206753060 (created 4h23m46s ago)
215 | ├── myes-super-133145198 (created 4s ago)
216 | ├── myes-update-207626186 (created 3m49s ago)
217 | └── myes-upgrade-521226614 (created 2m3s ago)
218 | ```
219 |
--------------------------------------------------------------------------------
/unit2/README.md:
--------------------------------------------------------------------------------
1 | ## unit 2: a stateful service
2 |
3 | This unit uses `elasticsearch` as base technology to showcase how to develop a stateful framework.
4 |
5 | ### [params.yaml](operator/params.yaml)
6 |
7 | The sample has one configuration parameter named `NODE_COUNT`, the number of nodes in the elasticsearch cluster, it defaults to `3`.
8 |
9 | ```yaml
10 | NODE_COUNT:
11 | default: "3"
12 | ```
13 |
14 | ### [templates](operator/templates)
15 |
16 | The sample has two resource templates, one is of type `Service` the other of type `StatefulSet`.
17 |
18 | StatefulSets currently require a [Headless Service](https://kubernetes.io/docs/concepts/services-networking/service/#headless-services) to be responsible for the network identity of the Pods.
19 |
20 | The following shows the resource template for the `Headless Service`.
21 |
22 | [service.yaml](operator/service.yaml)
23 | ```yaml
24 | kind: Service
25 | apiVersion: v1
26 | metadata:
27 | name: hs
28 | namespace: {{ .Namespace }}
29 | spec:
30 | selector:
31 | app: elastic
32 | ports:
33 | - protocol: TCP
34 | port: 9200
35 | clusterIP: None
36 | clusterIP: None
37 | ```
38 |
39 | A [StatefulSet](https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/) is used to manage a stateful service. It manages the deployment and scaling of a set of Pods , and provides guarantees about the ordering and uniqueness of these Pods. StatefulSet manages Pods that are based on an identical container spec, and maintains a sticky identity for each of the Pods. These pods are created from the same spec, but are not interchangeable. Each has a persistent identifier that it maintains across any rescheduling. A StatefulSet operates under the same pattern as any other Kubernetes Controller. You define your desired state in a StatefulSet object, and the StatefulSet controller makes any necessary updates to get there from the current state.
40 |
41 | The following shows the resource template for the `StaefulSet`.
42 |
43 | [node.yaml](operator/node.yaml)
44 | ```yaml
45 |
46 | node.yaml: |
47 | kind: StatefulSet
48 | apiVersion: apps/v1
49 | metadata:
50 | name: node
51 | namespace: {{ .Namespace }}
52 | spec:
53 | selector:
54 | matchLabels:
55 | app: elastic # has to match .spec.template.metadata.labels
56 | serviceName: {{ .Name }}-hs
57 | replicas: {{ .Params.NODE_COUNT }}
58 | template:
59 | metadata:
60 | labels:
61 | app: elastic # has to match .spec.selector.matchLabels
62 | spec:
63 | initContainers:
64 | - name: init-sysctl
65 | image: busybox
66 | command: ['sh', '-c', 'sysctl -w vm.max_map_count=262144']
67 | securityContext:
68 | privileged: true
69 | - name: volume-permissions
70 | image: busybox
71 | command: ['sh', '-c', 'chown -R 1000:1000 /usr/share/elasticsearch/data']
72 | volumeMounts:
73 | - name: data
74 | mountPath: /usr/share/elasticsearch/data
75 | terminationGracePeriodSeconds: 10
76 | containers:
77 | - name: elastic
78 | image: elasticsearch:7.0.0
79 | ports:
80 | - containerPort: 9200
81 | name: api
82 | - containerPort: 9300
83 | name:
84 | env:
85 | - name: cluster.name
86 | value: {{ .Name }}-cluster
87 | - name: discovery.seed_hosts
88 | value: {{ .Name }}-node-0.{{ .Name }}-hs,{{ .Name }}-node-1.{{ .Name }}-hs,{{ .Name }}-node-2.{{ .Name }}-hs
89 | - name: cluster.initial_master_nodes
90 | value: {{ .Name }}-node-0,{{ .Name }}-node-1,{{ .Name }}-node-2
91 | volumeMounts:
92 | - name: data
93 | mountPath: /usr/share/elasticsearch/data
94 | volumeClaimTemplates:
95 | - metadata:
96 | name: data
97 | spec:
98 | accessModes: [ "ReadWriteOnce" ]
99 | resources:
100 | requests:
101 | storage: 1Gi
102 | ```
103 |
104 | The StatefulSet has a `template` section that holds the specification of each pod in the set. It also has a `volumeClaimTemplate` for a persistent volume claim, this for a volume required by each pod in the set.
105 |
106 | The `template` section has a `spec` section which has the `initContainers` and `containers` specification of the pod.
107 |
108 | `containers` shows how in our sample the elasticsearch base technology is used. The container configuration is done via environment variables (alternatively could also be done via a config map as shown in unit 1). It also shows where the claimed volume is to be mounted in the container.
109 |
110 | `initContainers` come handy if the base technology requires some goofy stuff, like elasticsearch needs the `vm.max_map_count` setting, and also allow access to the mounted volume by the elasticsearch user that the container runs under (UID=1000, AND GID=1000).
111 |
112 | ### [operator.yaml](operator/operator.yaml)
113 |
114 | #### tasks
115 |
116 | The sample `deploy-task` applies the service and node template.
117 |
118 | ```yaml
119 | tasks:
120 | deploy-task:
121 | resources:
122 | - service.yaml
123 | - node.yaml
124 | ```
125 |
126 | #### plans
127 |
128 | The sample has a `deploy` plan with a `deploy-phase` and a `deploy-step`. From the `deploy-step` the `deploy-task` is referenced. This task gets executed when an instance is created using the operator.
129 |
130 | ```yaml
131 | plans:
132 | deploy:
133 | strategy: serial
134 | phases:
135 | - name: deploy-phase
136 | strategy: parallel
137 | steps:
138 | - name: deploy-step
139 | tasks:
140 | - deploy-task
141 | ```
142 |
143 |
144 | ### Run the framework instance
145 |
146 | If you haven't already then clone the `kudo-tutorial` repository.
147 |
148 | ```
149 | git clone https://github.com/realmbgl/kudo-tutorial.git
150 | ```
151 |
152 | From the `unit2/operator` folder use the following command to install the operator and create an instance
153 |
154 | ```
155 | kubectl kudo install . --instance myes
156 | ```
157 |
158 | Once the install is finished we should see the following pods.
159 | ```
160 | kubectl get pods
161 |
162 | NAME READY STATUS RESTARTS AGE
163 | myes-node-0 1/1 Running 0 121m
164 | myes-node-1 1/1 Running 0 121m
165 | myes-node-2 1/1 Running 0 121m
166 | ```
167 |
168 | Lets check on the elasticsearch clusters health.
169 |
170 | Exec into one of the POD's.
171 |
172 | ```
173 | kubectl exec -ti myes-node-2 bash
174 | ```
175 |
176 | Use the following curl command to check the health of the cluster.
177 |
178 | ```
179 | curl myes-node-0.myes-hs:9200/_cluster/health?pretty
180 | ```
181 |
182 | You should see the following output, showing the cluster status `green`.
183 |
184 | ```
185 | {
186 | "cluster_name" : "myes-cluster",
187 | "status" : "green",
188 | "timed_out" : false,
189 | "number_of_nodes" : 3,
190 | "number_of_data_nodes" : 3,
191 | "active_primary_shards" : 0,
192 | "active_shards" : 0,
193 | "relocating_shards" : 0,
194 | "initializing_shards" : 0,
195 | "unassigned_shards" : 0,
196 | "delayed_unassigned_shards" : 0,
197 | "number_of_pending_tasks" : 0,
198 | "number_of_in_flight_fetch" : 0,
199 | "task_max_waiting_in_queue_millis" : 0,
200 | "active_shards_percent_as_number" : 100.0
201 | }
202 | ```
203 |
204 |
205 | ### Update the framework instance to scale to 4 nodes
206 |
207 | Lets increase the `NODE_COUNT` to `4` using the following command.
208 |
209 | ```
210 | kubectl kudo update --instance myes -p NODE_COUNT=4
211 | ```
212 |
213 | Once the update is finished we should see an additional pod `myes-node-3`.
214 |
215 | ```
216 | kubectl get pods
217 |
218 | NAME READY STATUS RESTARTS AGE
219 | myes-node-0 1/1 Running 0 128m
220 | myes-node-1 1/1 Running 0 128m
221 | myes-node-2 1/1 Running 0 127m
222 | myes-node-3 1/1 Running 0 29s
223 | ```
224 |
--------------------------------------------------------------------------------
/unit4/README.md:
--------------------------------------------------------------------------------
1 | ## unit 4: custom plans
2 |
3 | This unit builds on `unit 2` showcasing custom backup and restore plans using `elasticsearch` [snapshot and restore](https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-snapshots.html). There are various repository plugins for elasticsearch snapshot restore, in this unit we are using the [repository-s3 plugin](https://www.elastic.co/guide/en/elasticsearch/plugins/7.0/repository-s3.html). The unit requires an S3 compatible store, we use `MinIO`. The `MinIO operator` install you can find [here](https://github.com/minio/minio-operator).
4 |
5 |
6 | ### [params.yaml](operator/params.yaml)
7 |
8 | The sample has four additional configuration parameters. The `S3_ACCESS_KEY`, `S3_SECRET_KEY`, and `S3_ENDPOINT` the defaults are set for the `MinIO` instance. There is also the `RESTORE_SNAPSHOT_ID` that we will have to set before running the restore plan.
9 |
10 | ```yaml
11 | NODE_COUNT:
12 | default: "3"
13 | S3_ACCESS_KEY:
14 | default: "minio"
15 | S3_SECRET_KEY:
16 | default: "minio123"
17 | S3_ENDPOINT:
18 | default: "minio:9000"
19 | RESTORE_SNAPSHOT_ID:
20 | default: ""
21 | ```
22 |
23 | ### [templates](operator/templates)
24 |
25 | The `container` section in the `node.yaml` template now has an explicit command setting that we need to install the `repository-s3` plugin, and to add `ACCESS_KEY` and `SECRET_KEY` to the `elasticsearch-keystore`.
26 |
27 | [node.yaml](operator/node.yaml)
28 | ```yaml
29 | ...
30 | containers:
31 | - name: elastic
32 | image: elasticsearch:7.0.0
33 | command:
34 | - sh
35 | - -c
36 | - |
37 | /usr/share/elasticsearch/bin/elasticsearch-plugin install repository-s3 -b;
38 | /usr/share/elasticsearch/bin/elasticsearch-keystore create
39 | echo {{ .Params.S3_ACCESS_KEY }} | /usr/share/elasticsearch/bin/elasticsearch-keystore add --stdin s3.client.default.access_key;
40 | echo {{ .Params.S3_SECRET_KEY }} | /usr/share/elasticsearch/bin/elasticsearch-keystore add --stdin s3.client.default.secret_key;
41 | /usr/local/bin/docker-entrypoint.sh eswrapper
42 | ...
43 | ```
44 |
45 |
46 | There are two additional resource templates in the framework implementation, they are both of type `Job`.
47 |
48 | The `backup` template does the elasticsearch `snapshot`.
49 |
50 | [backup.yaml](operator/backup.yaml)
51 | ```yaml
52 | apiVersion: batch/v1
53 | kind: Job
54 | metadata:
55 | name: {{ .PlanName }}-job
56 | namespace: default
57 | spec:
58 | template:
59 | metadata:
60 | name: {{ .PlanName }}-job
61 | spec:
62 | restartPolicy: OnFailure
63 | containers:
64 | - name: backup
65 | image: centos:7
66 | command:
67 | - sh
68 | - -c
69 | - |
70 | curl -X PUT "myes-node-0.myes-hs:9200/_snapshot/my_s3_repository" -H 'Content-Type: application/json' -d'
71 | {
72 | "type": "s3",
73 | "settings": {
74 | "bucket": "es-bucket",
75 | "endpoint": "{{ .Params.S3_ENDPOINT }}",
76 | "protocol": "http"
77 | }
78 | }
79 | ';
80 | TS=$(date +%s);
81 | curl -X PUT "myes-node-0.myes-hs:9200/_snapshot/my_s3_repository/snapshot_$TS?wait_for_completion=true&pretty"
82 | ```
83 |
84 | The `restore` template does the elasticsearch `restore`.
85 |
86 | [restore.yaml](operator/restore.yaml)
87 | ```yaml
88 | apiVersion: batch/v1
89 | kind: Job
90 | metadata:
91 | name: {{ .PlanName }}-job
92 | namespace: default
93 | spec:
94 | template:
95 | metadata:
96 | name: {{ .PlanName }}-job
97 | spec:
98 | restartPolicy: OnFailure
99 | containers:
100 | - name: restore
101 | image: centos:7
102 | command:
103 | - sh
104 | - -c
105 | - |
106 | curl -X POST "myes-node-0.myes-hs:9200/_snapshot/my_s3_repository/{{ .Params.RESTORE_SNAPSHOT_ID }}/_restore?pretty"
107 | ```
108 |
109 | ### [operator.yaml](operator/operator.yaml)
110 |
111 | #### tasks
112 |
113 | The `backup-task` applies the backup template, and the `restore-task` applies the restore template.
114 |
115 | ```yaml
116 | tasks:
117 | ...
118 | backup-task:
119 | resources:
120 | - backup.yaml
121 | restore-task:
122 | resources:
123 | - restore.yaml
124 | ```
125 |
126 | #### plans
127 |
128 | This sample has two custom plans. The `backup` and a `restore` plan.
129 |
130 | ```yaml
131 | backup:
132 | strategy: serial
133 | phases:
134 | - name: backup-phase
135 | strategy: serial
136 | steps:
137 | - name: backup-step
138 | tasks:
139 | - backup-task
140 | restore:
141 | strategy: serial
142 | phases:
143 | - name: restore-phase
144 | strategy: serial
145 | steps:
146 | - name: restore-step
147 | tasks:
148 | - restore-task
149 | ```
150 |
151 |
152 | ### Run the framework instance
153 |
154 | If you haven't already then clone the `kudo-tutorial` repository.
155 |
156 | ```
157 | git clone https://github.com/realmbgl/kudo-tutorial.git
158 | ```
159 |
160 | From the `unit4/operator` folder use the following command to run the instance.
161 |
162 | ```
163 | kubectl kudo install . --instance myes
164 | ```
165 |
166 |
167 | ### Add some data to the elasticsearch cluster
168 |
169 | Exec into one of the POD's.
170 |
171 | ```
172 | kubectl exec -ti myes-node-2 bash
173 | ```
174 |
175 | Lets add some data.
176 |
177 | ```
178 | curl -X POST "myes-node-0.myes-hs:9200/twitter/_doc/" -H 'Content-Type: application/json' -d'
179 | {
180 | "user" : "kimchy",
181 | "post_date" : "2009-11-15T14:12:12",
182 | "message" : "trying out Elasticsearch"
183 | }
184 | '
185 | ```
186 |
187 | Lets search for it.
188 |
189 | ```
190 | curl -X GET "myes-node-0.myes-hs:9200/twitter/_search?q=user:kimchy&pretty"
191 | ```
192 |
193 | You should see the following output.
194 |
195 | ```
196 | {
197 | "took" : 6,
198 | "timed_out" : false,
199 | "_shards" : {
200 | "total" : 1,
201 | "successful" : 1,
202 | "skipped" : 0,
203 | "failed" : 0
204 | },
205 | "hits" : {
206 | "total" : {
207 | "value" : 1,
208 | "relation" : "eq"
209 | },
210 | "max_score" : 0.2876821,
211 | "hits" : [
212 | {
213 | "_index" : "twitter",
214 | "_type" : "_doc",
215 | "_id" : "n18aemoBCj0qv5VrMWv2",
216 | "_score" : 0.2876821,
217 | "_source" : {
218 | "user" : "kimchy",
219 | "post_date" : "2009-11-15T14:12:12",
220 | "message" : "trying out Elasticsearch"
221 | }
222 | }
223 | ]
224 | }
225 | }
226 | ```
227 |
228 |
229 | ### Run the backup plan
230 |
231 | Before doing this the first time you need to go to your `MinIO` instance and create a bucket named `es-bucket` with policy `Read and Write`. You can do this via the `MinIO` console.
232 |
233 | Enable localhost access to the `MinIO` service.
234 |
235 | ```
236 | kubectl port-forward service/minio 9000
237 | ```
238 |
239 | [Click](http://localhost:9000/) to access the `MinIO console` from your browser.
240 |
241 | Next apply the backup plan from the `unit4` folder.
242 |
243 | ```
244 | kubectl apply -f ../backup-restore/backup.yaml
245 | ```
246 |
247 | Check the logs of the backup job for the id of the snapshot that got created. Should look like follows, so the id for this backup is `snapshot_1557432189`.
248 |
249 | ```
250 | {
251 | "snapshot" : {
252 | "snapshot" : "snapshot_1557432189",
253 | "uuid" : "3klj2YoBToiGmskPsKukuA",
254 | "version_id" : 7000099,
255 | "version" : "7.0.0",
256 | "indices" : [
257 | "twitter"
258 | ],
259 | "include_global_state" : true,
260 | "state" : "SUCCESS",
261 | "start_time" : "2019-05-09T20:03:09.810Z",
262 | "start_time_in_millis" : 1557432189810,
263 | "end_time" : "2019-05-09T20:03:10.053Z",
264 | "end_time_in_millis" : 1557432190053,
265 | "duration_in_millis" : 243,
266 | "failures" : [ ],
267 | "shards" : {
268 | "total" : 1,
269 | "failed" : 0,
270 | "successful" : 1
271 | }
272 | }
273 | }
274 | ```
275 |
276 |
277 | ### Run the restore plan
278 |
279 | Before doing this lets delete the data that we stored earlier.
280 |
281 | Exec into one of the POD's.
282 |
283 | ```
284 | kubectl exec -ti myes-node-2 bash
285 | ```
286 |
287 | From there we delete the whole index.
288 |
289 | ```
290 | curl -X DELETE "myes-node-0.myes-hs:9200/twitter"
291 | ```
292 |
293 | Back to the `unit4/operator` folder.
294 |
295 | Since you can't pass input at the moment when applying a PlanExecution, we need to update the instance configuration with the snapshot id to use for restore.
296 |
297 | ```
298 | kubectl patch instance myes -p '{"spec":{"parameters":{"RESTORE_SNAPSHOT_ID":"snapshot_1562101091"}}}' --type=merge
299 | ```
300 |
301 | Next apply the restore plan.
302 |
303 | ```
304 | kubectl apply -f ../backup-restore/restore.yaml
305 | ```
306 |
307 | Lets check that the data is back.
308 |
309 | Exec into one of the POD's.
310 |
311 | ```
312 | kubectl exec -ti myes-node-2 bash
313 | ```
314 |
315 | Lets search for the data. You should see the same JSON document that we saw earlier.
316 |
317 | ```
318 | curl -X GET "myes-node-0.myes-hs:9200/twitter/_search?q=user:kimchy&pretty"
319 | ```
320 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | Apache License
2 | Version 2.0, January 2004
3 | http://www.apache.org/licenses/
4 |
5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
6 |
7 | 1. Definitions.
8 |
9 | "License" shall mean the terms and conditions for use, reproduction,
10 | and distribution as defined by Sections 1 through 9 of this document.
11 |
12 | "Licensor" shall mean the copyright owner or entity authorized by
13 | the copyright owner that is granting the License.
14 |
15 | "Legal Entity" shall mean the union of the acting entity and all
16 | other entities that control, are controlled by, or are under common
17 | control with that entity. For the purposes of this definition,
18 | "control" means (i) the power, direct or indirect, to cause the
19 | direction or management of such entity, whether by contract or
20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the
21 | outstanding shares, or (iii) beneficial ownership of such entity.
22 |
23 | "You" (or "Your") shall mean an individual or Legal Entity
24 | exercising permissions granted by this License.
25 |
26 | "Source" form shall mean the preferred form for making modifications,
27 | including but not limited to software source code, documentation
28 | source, and configuration files.
29 |
30 | "Object" form shall mean any form resulting from mechanical
31 | transformation or translation of a Source form, including but
32 | not limited to compiled object code, generated documentation,
33 | and conversions to other media types.
34 |
35 | "Work" shall mean the work of authorship, whether in Source or
36 | Object form, made available under the License, as indicated by a
37 | copyright notice that is included in or attached to the work
38 | (an example is provided in the Appendix below).
39 |
40 | "Derivative Works" shall mean any work, whether in Source or Object
41 | form, that is based on (or derived from) the Work and for which the
42 | editorial revisions, annotations, elaborations, or other modifications
43 | represent, as a whole, an original work of authorship. For the purposes
44 | of this License, Derivative Works shall not include works that remain
45 | separable from, or merely link (or bind by name) to the interfaces of,
46 | the Work and Derivative Works thereof.
47 |
48 | "Contribution" shall mean any work of authorship, including
49 | the original version of the Work and any modifications or additions
50 | to that Work or Derivative Works thereof, that is intentionally
51 | submitted to Licensor for inclusion in the Work by the copyright owner
52 | or by an individual or Legal Entity authorized to submit on behalf of
53 | the copyright owner. For the purposes of this definition, "submitted"
54 | means any form of electronic, verbal, or written communication sent
55 | to the Licensor or its representatives, including but not limited to
56 | communication on electronic mailing lists, source code control systems,
57 | and issue tracking systems that are managed by, or on behalf of, the
58 | Licensor for the purpose of discussing and improving the Work, but
59 | excluding communication that is conspicuously marked or otherwise
60 | designated in writing by the copyright owner as "Not a Contribution."
61 |
62 | "Contributor" shall mean Licensor and any individual or Legal Entity
63 | on behalf of whom a Contribution has been received by Licensor and
64 | subsequently incorporated within the Work.
65 |
66 | 2. Grant of Copyright License. Subject to the terms and conditions of
67 | this License, each Contributor hereby grants to You a perpetual,
68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
69 | copyright license to reproduce, prepare Derivative Works of,
70 | publicly display, publicly perform, sublicense, and distribute the
71 | Work and such Derivative Works in Source or Object form.
72 |
73 | 3. Grant of Patent License. Subject to the terms and conditions of
74 | this License, each Contributor hereby grants to You a perpetual,
75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
76 | (except as stated in this section) patent license to make, have made,
77 | use, offer to sell, sell, import, and otherwise transfer the Work,
78 | where such license applies only to those patent claims licensable
79 | by such Contributor that are necessarily infringed by their
80 | Contribution(s) alone or by combination of their Contribution(s)
81 | with the Work to which such Contribution(s) was submitted. If You
82 | institute patent litigation against any entity (including a
83 | cross-claim or counterclaim in a lawsuit) alleging that the Work
84 | or a Contribution incorporated within the Work constitutes direct
85 | or contributory patent infringement, then any patent licenses
86 | granted to You under this License for that Work shall terminate
87 | as of the date such litigation is filed.
88 |
89 | 4. Redistribution. You may reproduce and distribute copies of the
90 | Work or Derivative Works thereof in any medium, with or without
91 | modifications, and in Source or Object form, provided that You
92 | meet the following conditions:
93 |
94 | (a) You must give any other recipients of the Work or
95 | Derivative Works a copy of this License; and
96 |
97 | (b) You must cause any modified files to carry prominent notices
98 | stating that You changed the files; and
99 |
100 | (c) You must retain, in the Source form of any Derivative Works
101 | that You distribute, all copyright, patent, trademark, and
102 | attribution notices from the Source form of the Work,
103 | excluding those notices that do not pertain to any part of
104 | the Derivative Works; and
105 |
106 | (d) If the Work includes a "NOTICE" text file as part of its
107 | distribution, then any Derivative Works that You distribute must
108 | include a readable copy of the attribution notices contained
109 | within such NOTICE file, excluding those notices that do not
110 | pertain to any part of the Derivative Works, in at least one
111 | of the following places: within a NOTICE text file distributed
112 | as part of the Derivative Works; within the Source form or
113 | documentation, if provided along with the Derivative Works; or,
114 | within a display generated by the Derivative Works, if and
115 | wherever such third-party notices normally appear. The contents
116 | of the NOTICE file are for informational purposes only and
117 | do not modify the License. You may add Your own attribution
118 | notices within Derivative Works that You distribute, alongside
119 | or as an addendum to the NOTICE text from the Work, provided
120 | that such additional attribution notices cannot be construed
121 | as modifying the License.
122 |
123 | You may add Your own copyright statement to Your modifications and
124 | may provide additional or different license terms and conditions
125 | for use, reproduction, or distribution of Your modifications, or
126 | for any such Derivative Works as a whole, provided Your use,
127 | reproduction, and distribution of the Work otherwise complies with
128 | the conditions stated in this License.
129 |
130 | 5. Submission of Contributions. Unless You explicitly state otherwise,
131 | any Contribution intentionally submitted for inclusion in the Work
132 | by You to the Licensor shall be under the terms and conditions of
133 | this License, without any additional terms or conditions.
134 | Notwithstanding the above, nothing herein shall supersede or modify
135 | the terms of any separate license agreement you may have executed
136 | with Licensor regarding such Contributions.
137 |
138 | 6. Trademarks. This License does not grant permission to use the trade
139 | names, trademarks, service marks, or product names of the Licensor,
140 | except as required for reasonable and customary use in describing the
141 | origin of the Work and reproducing the content of the NOTICE file.
142 |
143 | 7. Disclaimer of Warranty. Unless required by applicable law or
144 | agreed to in writing, Licensor provides the Work (and each
145 | Contributor provides its Contributions) on an "AS IS" BASIS,
146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147 | implied, including, without limitation, any warranties or conditions
148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149 | PARTICULAR PURPOSE. You are solely responsible for determining the
150 | appropriateness of using or redistributing the Work and assume any
151 | risks associated with Your exercise of permissions under this License.
152 |
153 | 8. Limitation of Liability. In no event and under no legal theory,
154 | whether in tort (including negligence), contract, or otherwise,
155 | unless required by applicable law (such as deliberate and grossly
156 | negligent acts) or agreed to in writing, shall any Contributor be
157 | liable to You for damages, including any direct, indirect, special,
158 | incidental, or consequential damages of any character arising as a
159 | result of this License or out of the use or inability to use the
160 | Work (including but not limited to damages for loss of goodwill,
161 | work stoppage, computer failure or malfunction, or any and all
162 | other commercial damages or losses), even if such Contributor
163 | has been advised of the possibility of such damages.
164 |
165 | 9. Accepting Warranty or Additional Liability. While redistributing
166 | the Work or Derivative Works thereof, You may choose to offer,
167 | and charge a fee for, acceptance of support, warranty, indemnity,
168 | or other liability obligations and/or rights consistent with this
169 | License. However, in accepting such obligations, You may act only
170 | on Your own behalf and on Your sole responsibility, not on behalf
171 | of any other Contributor, and only if You agree to indemnify,
172 | defend, and hold each Contributor harmless for any liability
173 | incurred by, or claims asserted against, such Contributor by reason
174 | of your accepting any such warranty or additional liability.
175 |
176 | END OF TERMS AND CONDITIONS
177 |
178 | APPENDIX: How to apply the Apache License to your work.
179 |
180 | To apply the Apache License to your work, attach the following
181 | boilerplate notice, with the fields enclosed by brackets "[]"
182 | replaced with your own identifying information. (Don't include
183 | the brackets!) The text should be enclosed in the appropriate
184 | comment syntax for the file format. We also recommend that a
185 | file or class name and description of purpose be included on the
186 | same "printed page" as the copyright notice for easier
187 | identification within third-party archives.
188 |
189 | Copyright [yyyy] [name of copyright owner]
190 |
191 | Licensed under the Apache License, Version 2.0 (the "License");
192 | you may not use this file except in compliance with the License.
193 | You may obtain a copy of the License at
194 |
195 | http://www.apache.org/licenses/LICENSE-2.0
196 |
197 | Unless required by applicable law or agreed to in writing, software
198 | distributed under the License is distributed on an "AS IS" BASIS,
199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200 | See the License for the specific language governing permissions and
201 | limitations under the License.
202 |
--------------------------------------------------------------------------------